Skip to content

Open-source AI teammates in Slack, Teams, Discord, Telegram and Mattermost. Self-host it, with any agent harness and any model.

fix(bridges): serialise runtime-indicator refresh and repositioning (CHOO-1104)

Louis Amaudruzc363260a36d3
diff --git a/core/switch_core/bridges/collaboration/adapter.py b/core/switch_core/bridges/collaboration/adapter.py
index e48d00345..769bace0b 100644
--- a/core/switch_core/bridges/collaboration/adapter.py
+++ b/core/switch_core/bridges/collaboration/adapter.py
@@ -1,5 +1,6 @@
from __future__ import annotations
+import asyncio
import logging
from abc import ABC, abstractmethod
from collections.abc import Awaitable, Callable
@@ -53,6 +54,9 @@ class CollaborationAdapter(ABC):
# typing-indicator default leaves them empty.
self._working_msg: dict[tuple[str, str], LiveRuntimeIndicator] = {}
self._input_pings: dict[tuple[str, str], list[str]] = {}
+ # One lock per (channel_id, agent_name). Every mutation of the entries
+ # above happens under it — see _runtime_lock.
+ self._runtime_locks: dict[tuple[str, str], asyncio.Lock] = {}
def set_max_attachment_bytes(self, max_bytes: int) -> None:
self._max_attachment_bytes = max_bytes
@@ -178,6 +182,20 @@ class CollaborationAdapter(ABC):
self, channel_id: str, sender_name: str, is_typing: bool
) -> None: ...
+ def _runtime_lock(self, channel_id: str, agent_name: str) -> asyncio.Lock:
+ """The lock serialising runtime-indicator work for one agent in one
+ channel.
+
+ The indicator is mutated from two independent places — the periodic
+ activity refresh and a reposition triggered by new traffic — and each
+ reads the tracked message, awaits a platform call, then writes it back.
+ Left to interleave, the later write restores a superseded message ref:
+ the entry then names a message that has just been deleted while the one
+ actually on screen is referenced by nothing, so the end-of-turn clear
+ cannot remove it and it stays in the channel for good.
+ """
+ return self._runtime_locks.setdefault((channel_id, agent_name), asyncio.Lock())
+
async def apply_runtime_state(
self,
channel_id: str,
@@ -188,6 +206,39 @@ class CollaborationAdapter(ABC):
thread_root_id: str | None,
deeplink_url: str | None = None,
detail: str | None = None,
+ ) -> None:
+ """Serialise against any other runtime-indicator work for this agent,
+ then apply the state. Adapters override ``_apply_runtime_state``."""
+ async with self._runtime_lock(channel_id, agent_name):
+ await self._apply_runtime_state(
+ channel_id,
+ agent_name,
+ state,
+ notify_user=notify_user,
+ thread_root_id=thread_root_id,
+ deeplink_url=deeplink_url,
+ detail=detail,
+ )
+
+ async def reposition_runtime_state(
+ self, channel_id: str, agent_name: str, thread_root_id: str | None
+ ) -> None:
+ """Serialise against any other runtime-indicator work for this agent,
+ then move the indicator. Adapters override
+ ``_reposition_runtime_state``."""
+ async with self._runtime_lock(channel_id, agent_name):
+ await self._reposition_runtime_state(channel_id, agent_name, thread_root_id)
+
+ async def _apply_runtime_state(
+ self,
+ channel_id: str,
+ agent_name: str,
+ state: str,
+ *,
+ notify_user: str | None,
+ thread_root_id: str | None,
+ deeplink_url: str | None = None,
+ detail: str | None = None,
) -> None:
"""Surface a switchdash-managed agent's runtime state on the channel.
@@ -229,7 +280,7 @@ class CollaborationAdapter(ABC):
if posted_channel == channel_id
]
- async def reposition_runtime_state(
+ async def _reposition_runtime_state(
self, channel_id: str, agent_name: str, thread_root_id: str | None
) -> None:
"""Move the agent's live runtime indicator to follow the latest message.
@@ -245,6 +296,9 @@ class CollaborationAdapter(ABC):
out to the channel root) rather than being stranded in whichever thread
the turn happened to start in.
+ Runs under the agent's runtime lock, so the tracked indicator cannot be
+ cleared or refreshed part-way through.
+
Adapters that render runtime state as a typing indicator have nothing
positional to move, so the default does nothing.
"""
@@ -263,14 +317,6 @@ class CollaborationAdapter(ABC):
)
return
- if self._working_msg.get(key) is not live:
- # The turn ended (or another move landed) while this post was in
- # flight, so whoever cleared it has already removed the message this
- # one replaces. Nothing will ever clear the replacement, so take it
- # back down rather than strand it in the channel.
- await self._remove_runtime_indicator(channel_id, ref)
- return
-
self._working_msg[key] = replace(
live, message_ref=ref, thread_root_id=thread_root_id
)
diff --git a/core/switch_core/bridges/collaboration/bridge_core.py b/core/switch_core/bridges/collaboration/bridge_core.py
index ac38417cf..2f71453d1 100644
--- a/core/switch_core/bridges/collaboration/bridge_core.py
+++ b/core/switch_core/bridges/collaboration/bridge_core.py
@@ -1418,7 +1418,12 @@ class BridgeCore:
)
async def _run_indicator_move(self, key: tuple[str, str]) -> None:
- self._indicator_move_timers.pop(key, None)
+ # An anchor-driven move runs immediately rather than through the timer,
+ # so a coalescing window opened by outbound traffic may still be
+ # pending; it would otherwise fire a second, redundant move.
+ timer = self._indicator_move_timers.pop(key, None)
+ if timer is not None:
+ timer.cancel()
thread_root_id = self._indicator_move_targets.pop(key, None)
channel_id, agent_name = key
try:
diff --git a/core/switch_core/bridges/collaboration/discord/adapter.py b/core/switch_core/bridges/collaboration/discord/adapter.py
index 8db0ea3e3..cddf6ed97 100644
--- a/core/switch_core/bridges/collaboration/discord/adapter.py
+++ b/core/switch_core/bridges/collaboration/discord/adapter.py
@@ -442,7 +442,7 @@ class DiscordAdapter(CollaborationAdapter):
# ── Runtime state ────────────────────────────────────────────────────────
- async def apply_runtime_state(
+ async def _apply_runtime_state(
self,
channel_id: str,
agent_name: str,
diff --git a/core/switch_core/bridges/collaboration/mattermost/adapter.py b/core/switch_core/bridges/collaboration/mattermost/adapter.py
index 90ee66022..6fc82dbcd 100644
--- a/core/switch_core/bridges/collaboration/mattermost/adapter.py
+++ b/core/switch_core/bridges/collaboration/mattermost/adapter.py
@@ -440,7 +440,7 @@ class MattermostAdapter(CollaborationAdapter):
# ── Runtime state ──────────────────────────────────────────────────────────
- async def apply_runtime_state(
+ async def _apply_runtime_state(
self,
channel_id: str,
agent_name: str,
@@ -503,7 +503,7 @@ class MattermostAdapter(CollaborationAdapter):
agent_name, post_id, self.translate_outbound("✓ input received")
)
- async def reposition_runtime_state(
+ async def _reposition_runtime_state(
self, channel_id: str, agent_name: str, thread_root_id: str | None
) -> None:
"""Move the indicator only when Mattermost will truly delete the old one.
@@ -535,12 +535,6 @@ class MattermostAdapter(CollaborationAdapter):
)
return
- if self._working_msg.get(key) is not live:
- # The turn ended while the delete was in flight. The old post is
- # already gone and the entry with it, so there is nothing left to
- # move — reposting now would strand an indicator no one will clear.
- return
-
ref = await self.send_message(channel_id, agent_name, live.body, thread_root_id)
if ref is None:
# The old post is already gone, so there is nothing to fall back to.
@@ -553,12 +547,6 @@ class MattermostAdapter(CollaborationAdapter):
)
return
- if self._working_msg.get(key) is not live:
- # Same race, one step later: the replacement has no owner now, so
- # take it back down rather than leave it in the channel.
- await self._permanent_delete(ref)
- return
-
self._working_msg[key] = replace(
live, message_ref=ref, thread_root_id=thread_root_id
)
diff --git a/core/switch_core/bridges/collaboration/slack/adapter.py b/core/switch_core/bridges/collaboration/slack/adapter.py
index c7f5ca59c..bfa6def8d 100644
--- a/core/switch_core/bridges/collaboration/slack/adapter.py
+++ b/core/switch_core/bridges/collaboration/slack/adapter.py
@@ -444,7 +444,7 @@ class SlackAdapter(CollaborationAdapter):
# ── Runtime state ──────────────────────────────────────────────────────────
- async def apply_runtime_state(
+ async def _apply_runtime_state(
self,
channel_id: str,
agent_name: str,
diff --git a/core/switch_core/bridges/collaboration/teams/adapter.py b/core/switch_core/bridges/collaboration/teams/adapter.py
index 19f281a6a..9ec868e89 100644
--- a/core/switch_core/bridges/collaboration/teams/adapter.py
+++ b/core/switch_core/bridges/collaboration/teams/adapter.py
@@ -531,7 +531,7 @@ class TeamsAdapter(CollaborationAdapter):
# ── Runtime state ──────────────────────────────────────────────────────────
- async def apply_runtime_state(
+ async def _apply_runtime_state(
self,
channel_id: str,
agent_name: str,
diff --git a/core/tests/switch_core/bridges/collaboration/test_mattermost_runtime_state.py b/core/tests/switch_core/bridges/collaboration/test_mattermost_runtime_state.py
index 1ae4eb5aa..31dd35ab4 100644
--- a/core/tests/switch_core/bridges/collaboration/test_mattermost_runtime_state.py
+++ b/core/tests/switch_core/bridges/collaboration/test_mattermost_runtime_state.py
@@ -37,6 +37,9 @@ class _Recorder:
self.hard_deletes: list[str] = []
self.patches: list[tuple[str, str]] = []
self.sends: list[tuple[str, str, str, str | None]] = []
+ # Post ids actually handed back, so a test can assert that everything
+ # posted was also removed.
+ self.sent_ids: list[str] = []
self._next_id = iter(["post-2", "post-3", "post-4"])
def install(self, adapter: MattermostAdapter) -> None:
@@ -54,7 +57,9 @@ class _Recorder:
thread_root_id: str | None = None,
) -> str | None:
self.sends.append((channel_id, sender_name, content, thread_root_id))
- return next(self._next_id)
+ ref = next(self._next_id)
+ self.sent_ids.append(ref)
+ return ref
adapter._permanent_delete = permanent_delete # type: ignore[method-assign]
adapter._patch_post_as = patch_post_as # type: ignore[method-assign]
@@ -155,47 +160,58 @@ def test_failed_repost_after_delete_drops_the_stale_ref() -> None:
assert ("chan-1", "worker") not in adapter._working_msg
-def test_a_turn_ending_mid_delete_abandons_the_move() -> None:
- # Mattermost deletes first, so a turn ending during that delete leaves
- # nothing to move — reposting anyway would strand an indicator forever.
+def test_a_turn_ending_during_a_move_leaves_nothing_behind() -> None:
+ # The move and the end-of-turn clear share the agent's runtime lock, so
+ # they cannot interleave: whichever runs second sees the other's result.
+ # Either way the channel ends up with no indicator and none is tracked.
adapter = _adapter()
recorder = _Recorder(hard_delete_works=True)
recorder.install(adapter)
_seed_indicator(adapter)
- real_delete = adapter._permanent_delete
-
- async def delete_then_go_idle(post_id: str) -> bool:
- ok = await real_delete(post_id)
- adapter._working_msg.pop(("chan-1", "worker"), None)
- return ok
+ async def scenario() -> None:
+ await asyncio.gather(
+ adapter.reposition_runtime_state("chan-1", "worker", None),
+ adapter.apply_runtime_state(
+ "chan-1", "worker", "idle", notify_user=None, thread_root_id=None
+ ),
+ )
- adapter._permanent_delete = delete_then_go_idle # type: ignore[method-assign]
- _run(adapter.reposition_runtime_state("chan-1", "worker", None))
+ _run(scenario())
- assert recorder.sends == []
+ posted = {"post-1", *(ref for ref in recorder.sent_ids)}
+ assert posted <= set(recorder.hard_deletes)
assert ("chan-1", "worker") not in adapter._working_msg
-def test_a_turn_ending_mid_repost_takes_the_replacement_back_down() -> None:
+def test_a_refresh_during_a_move_does_not_strand_the_replacement() -> None:
+ # The reported leftover: the periodic activity refresh and the move both
+ # read-modify-write the tracked indicator. Serialised, the refresh edits
+ # whichever message is actually posted rather than restoring a dead ref.
adapter = _adapter()
recorder = _Recorder(hard_delete_works=True)
recorder.install(adapter)
_seed_indicator(adapter)
- real_send = adapter.send_message
-
- async def send_then_go_idle(*args: Any, **kwargs: Any) -> str | None:
- ref = await real_send(*args, **kwargs)
- adapter._working_msg.pop(("chan-1", "worker"), None)
- return ref
+ async def scenario() -> None:
+ await asyncio.gather(
+ adapter.reposition_runtime_state("chan-1", "worker", None),
+ adapter.apply_runtime_state(
+ "chan-1",
+ "worker",
+ "working",
+ notify_user=None,
+ thread_root_id=None,
+ detail="Ran tool post_message",
+ ),
+ )
- adapter.send_message = send_then_go_idle # type: ignore[method-assign]
- _run(adapter.reposition_runtime_state("chan-1", "worker", None))
+ _run(scenario())
- # post-1 was the move's own delete; post-2 is the orphan being cleaned up.
- assert recorder.hard_deletes == ["post-1", "post-2"]
- assert ("chan-1", "worker") not in adapter._working_msg
+ live = adapter._working_msg[("chan-1", "worker")]
+ posted = {"post-1", *recorder.sent_ids}
+ still_up = posted - set(recorder.hard_deletes)
+ assert still_up == {live.message_ref}
def test_idle_still_falls_back_to_a_terminal_marker() -> None:
diff --git a/core/tests/switch_core/bridges/collaboration/test_runtime_indicator_race.py b/core/tests/switch_core/bridges/collaboration/test_runtime_indicator_race.py
new file mode 100644
index 000000000..24836805f
--- /dev/null
+++ b/core/tests/switch_core/bridges/collaboration/test_runtime_indicator_race.py
@@ -0,0 +1,186 @@
+"""Moving the runtime indicator and refreshing it must not interleave.
+
+Two independent callers mutate ``_working_msg`` for the same agent: the
+periodic activity refresh (``apply_runtime_state`` with ``working``, every few
+seconds) and a reposition triggered by new traffic. Both read the entry, await
+a platform call, then write it back.
+
+Interleaved, the refresh's write lands after the move's and restores the
+superseded message ref. The entry then points at a message the move has just
+deleted, and the message the move posted is referenced by nothing — so the
+end-of-turn clear cannot remove it and it stays in the channel forever.
+
+The invariant each test asserts is the same: whatever is still posted on the
+platform is exactly what the adapter thinks is posted.
+"""
+
+from __future__ import annotations
+
+import asyncio
+from typing import Any
+
+from switch_core.bridges.collaboration.adapter import LiveRuntimeIndicator
+from switch_core.bridges.collaboration.slack.adapter import (
+ SlackAdapter,
+ SlackConnectionConfig,
+)
+
+CHANNEL = "chan-1"
+AGENT = "worker"
+KEY = (CHANNEL, AGENT)
+
+
+class _Platform:
+ """Records posts, edits and deletes, yielding once per call.
+
+ The single yield is what lets the two coroutines interleave at all — it
+ stands in for the network round trip each of these calls really makes.
+ """
+
+ def __init__(self, adapter: SlackAdapter, seeded_ref: str) -> None:
+ self.live: set[str] = {seeded_ref}
+ self.edits: list[tuple[str, str]] = []
+ self._next = iter(f"msg-{n}" for n in range(2, 20))
+
+ async def send_message(
+ channel_id: str,
+ sender_name: str,
+ content: str,
+ thread_root_id: str | None = None,
+ ) -> str | None:
+ await asyncio.sleep(0)
+ ref = next(self._next)
+ self.live.add(ref)
+ return ref
+
+ async def update_message(
+ channel_id: str, message_ref: str, new_content: str
+ ) -> None:
+ await asyncio.sleep(0)
+ self.edits.append((message_ref, new_content))
+
+ async def delete_message(channel_id: str, message_ref: str) -> None:
+ await asyncio.sleep(0)
+ self.live.discard(message_ref)
+
+ adapter.send_message = send_message # type: ignore[method-assign]
+ adapter.update_message = update_message # type: ignore[method-assign]
+ adapter.delete_message = delete_message # type: ignore[method-assign]
+
+
+def _adapter() -> tuple[SlackAdapter, _Platform]:
+ adapter = SlackAdapter(
+ config=SlackConnectionConfig(
+ bot_token="xoxb-test", app_token="xapp-test", workspace_id="T-test"
+ )
+ )
+ adapter._working_msg[KEY] = LiveRuntimeIndicator(
+ message_ref="msg-1",
+ body="⚙️ _Working on it…_",
+ thread_root_id=None,
+ )
+ return adapter, _Platform(adapter, "msg-1")
+
+
+def _refresh(adapter: SlackAdapter, detail: str) -> Any:
+ return adapter.apply_runtime_state(
+ CHANNEL,
+ AGENT,
+ "working",
+ notify_user=None,
+ thread_root_id=None,
+ detail=detail,
+ )
+
+
+def _assert_consistent(adapter: SlackAdapter, platform: _Platform) -> None:
+ live = adapter._working_msg.get(KEY)
+ tracked = {live.message_ref} if live is not None else set()
+ assert platform.live == tracked, (
+ f"platform still shows {sorted(platform.live)} but the adapter tracks "
+ f"{sorted(tracked)} — the difference is stranded and nothing will remove it"
+ )
+
+
+async def test_a_refresh_landing_during_a_move_does_not_strand_the_new_message() -> (
+ None
+):
+ # The reported bug: the refresh read the entry before the move rewrote it,
+ # so its write restored the old ref and orphaned the message the move
+ # posted. The turn's clear then removed the already-deleted one.
+ adapter, platform = _adapter()
+
+ await asyncio.gather(
+ adapter.reposition_runtime_state(CHANNEL, AGENT, None),
+ _refresh(adapter, "Ran tool post_message"),
+ )
+
+ _assert_consistent(adapter, platform)
+
+
+async def test_a_move_still_moves_when_a_refresh_races_it() -> None:
+ # The same interleaving in the other order silently abandons the move: the
+ # indicator stays where it was, which is the whole defect this feature
+ # exists to fix.
+ adapter, platform = _adapter()
+
+ await asyncio.gather(
+ _refresh(adapter, "Ran tool post_message"),
+ adapter.reposition_runtime_state(CHANNEL, AGENT, None),
+ )
+
+ live = adapter._working_msg[KEY]
+ assert live.message_ref != "msg-1", (
+ "the indicator was never repositioned — a concurrent activity refresh "
+ "cancelled the move"
+ )
+ _assert_consistent(adapter, platform)
+
+
+async def test_the_refresh_body_survives_a_concurrent_move() -> None:
+ # Whichever order they run in, the latest activity line must end up on the
+ # message that is actually still posted — not on a deleted one.
+ adapter, platform = _adapter()
+
+ await asyncio.gather(
+ adapter.reposition_runtime_state(CHANNEL, AGENT, None),
+ _refresh(adapter, "Editing foo.py"),
+ )
+
+ live = adapter._working_msg[KEY]
+ assert "Editing foo.py" in live.body
+ assert live.message_ref in platform.live
+
+
+async def test_a_burst_of_moves_and_refreshes_leaves_exactly_one_message() -> None:
+ # The steady state under load: several repositions and refreshes overlapping
+ # must still converge on a single tracked, still-posted indicator.
+ adapter, platform = _adapter()
+
+ await asyncio.gather(
+ *(adapter.reposition_runtime_state(CHANNEL, AGENT, None) for _ in range(4)),
+ *(_refresh(adapter, f"step {n}") for n in range(4)),
+ )
+
+ assert len(platform.live) == 1
+ _assert_consistent(adapter, platform)
+
+
+async def test_the_turn_end_clear_removes_everything() -> None:
+ # After a contended turn, going idle must leave the channel clean.
+ adapter, platform = _adapter()
+
+ await asyncio.gather(
+ adapter.reposition_runtime_state(CHANNEL, AGENT, None),
+ _refresh(adapter, "Ran tool post_message"),
+ )
+ await adapter.apply_runtime_state(
+ CHANNEL,
+ AGENT,
+ "idle",
+ notify_user=None,
+ thread_root_id=None,
+ )
+
+ assert platform.live == set()
+ assert KEY not in adapter._working_msg
diff --git a/core/tests/switch_core/bridges/collaboration/test_slack_adapter.py b/core/tests/switch_core/bridges/collaboration/test_slack_adapter.py
index eb096f2c2..1e2c189b8 100644
--- a/core/tests/switch_core/bridges/collaboration/test_slack_adapter.py
+++ b/core/tests/switch_core/bridges/collaboration/test_slack_adapter.py
@@ -50,6 +50,9 @@ class _FakeWebClient:
self.calls: list[dict[str, Any]] = []
self.deletes: list[dict[str, Any]] = []
self.updates: list[dict[str, Any]] = []
+ # ts values actually handed back, in order — lets a test assert that
+ # everything posted was also removed.
+ self.issued: list[str] = []
self._next_ts = iter(["999.9", "888.8", "777.7"])
async def chat_postMessage(self, **kwargs: Any) -> dict[str, str]:
@@ -58,6 +61,7 @@ class _FakeWebClient:
ts = next(self._next_ts)
except StopIteration:
ts = "000.0"
+ self.issued.append(ts)
return {"ts": ts}
async def chat_delete(self, **kwargs: Any) -> dict[str, bool]:
@@ -771,35 +775,30 @@ def test_reposition_is_a_noop_without_a_live_indicator() -> None:
assert fake.deletes == []
-def test_a_turn_ending_mid_move_does_not_strand_the_replacement() -> None:
- # The turn can go idle while the replacement post is still in flight. The
- # clear removes the message this move was replacing, so the replacement
- # would be left with nothing to ever clear it — it must be taken back down.
+def test_a_turn_ending_during_a_move_clears_what_the_move_left() -> None:
+ # A move and the end-of-turn clear cannot interleave: both run under the
+ # agent's runtime lock, so the clear sees the moved indicator rather than
+ # the message the move already deleted. Nothing is left in the channel.
adapter = _adapter()
fake = _FakeWebClient()
adapter._web_client = fake # type: ignore[assignment]
- _run(
- adapter.apply_runtime_state(
+ async def scenario() -> None:
+ await adapter.apply_runtime_state(
"C123", "agent-bot", "working", notify_user=None, thread_root_id=None
)
- )
-
- real_send = adapter.send_message
-
- async def send_then_go_idle(*args: Any, **kwargs: Any) -> str | None:
- ref = await real_send(*args, **kwargs)
- await adapter.apply_runtime_state(
- "C123", "agent-bot", "idle", notify_user=None, thread_root_id=None
+ await asyncio.gather(
+ adapter.reposition_runtime_state("C123", "agent-bot", None),
+ adapter.apply_runtime_state(
+ "C123", "agent-bot", "idle", notify_user=None, thread_root_id=None
+ ),
)
- return ref
- adapter.send_message = send_then_go_idle # type: ignore[method-assign]
- _run(adapter.reposition_runtime_state("C123", "agent-bot", None))
+ _run(scenario())
- # Both the original and the orphaned replacement are gone, and nothing is
- # left tracked for a turn that has ended.
- assert {d["ts"] for d in fake.deletes} == {"999.9", "888.8"}
+ # Every message the adapter posted was deleted again — whichever order the
+ # two ran in, the channel ends up empty and nothing is left tracked.
+ assert set(fake.issued) == {d["ts"] for d in fake.deletes}
assert ("C123", "agent-bot") not in adapter._working_msg
diff --git a/core/uv.lock b/core/uv.lock
index 62611169c..5d720864b 100644
--- a/core/uv.lock
+++ b/core/uv.lock
@@ -2433,7 +2433,7 @@ wheels = [
[[package]]
name = "switch-core"
-version = "0.12.0"
+version = "0.12.1"
source = { editable = "." }
dependencies = [
{ name = "aiohttp" },