Open-source AI teammates in Slack, Teams, Discord, Telegram and Mattermost. Self-host it, with any agent harness and any model.
Python agent-orchestrationaiai-agentschatopsdiscordlangchainmcpmicrosoft-teamsmulti-agentopen-sourceopenaiself-hostedslack
Retry attaching Discord bridges, relay Teams subscription removals, and stop the Discord identity errors at boot (#890)
Abel Dantas3bd3b801f688
diff --git a/core/switch_core/bridges/collaboration/adapter.py b/core/switch_core/bridges/collaboration/adapter.pyindex 098436ba1..c70d98b08 100644--- a/core/switch_core/bridges/collaboration/adapter.py+++ b/core/switch_core/bridges/collaboration/adapter.py@@ -393,6 +393,11 @@ class SupportsSharedConnection(Protocol):def set_on_attached(self, callback: Callable[[], None]) -> None: ...+ def awaiting_shared_connection(self) -> bool:+ """Whether this bridge runs on the shared connection and has not been+ attached to it yet, so nothing that reaches the platform can run."""+ ...+class PlatformAdapter(ABC):#: The platform as a person names it — "Microsoft Teams", not "teams".diff --git a/core/switch_core/bridges/collaboration/collaboration_core.py b/core/switch_core/bridges/collaboration/collaboration_core.pyindex d2e93de5f..e5625c15a 100644--- a/core/switch_core/bridges/collaboration/collaboration_core.py+++ b/core/switch_core/bridges/collaboration/collaboration_core.py@@ -531,7 +531,11 @@ class CollaborationCore:# restarts one. Messages do not depend on it: an agent is addressable# by name whether or not its platform identity exists yet. Every# process provisions, the adapter deciding how much a quiet one needs.- self._identity_task = asyncio.create_task(self._run_agent_identities())+ # Not yet on a shared connection that is not attached: every agent's+ # call would fail, and attaching provisions them all. Guarded, since+ # an attach while the adapter started may have begun a run already.+ if not self._awaiting_shared_connection():+ self._provision_identities_on_attach()# Not awaited either: one platform read per channel, and nothing waits# on it. Only where the bridge is held: one process is enough.if self._active:@@ -550,7 +554,16 @@ class CollaborationCore:let go of the deleted ones'. A run already going reads the agentsagain once it is done, in case it read them before the change."""self._identities_stale = True- self._provision_identities_on_attach()+ if not self._awaiting_shared_connection():+ self._provision_identities_on_attach()++ def _awaiting_shared_connection(self) -> bool:+ """Whether the bridge waits to be attached to its platform's shared+ connection, which provisions the agents' identities once it is."""+ return (+ isinstance(self._adapter, SupportsSharedConnection)+ and self._adapter.awaiting_shared_connection()+ )async def stop(self) -> None:try:@@ -646,22 +659,24 @@ class CollaborationCore:self._human_user_ids.add(client.transport_user_id)def _provision_identities_on_attach(self) -> None:- """Re-run identity provisioning when a shared connection attaches.-- The start-time run failed for a shared bridge because it had no- connection yet. Attaching gives it one, so provision now — a fresh- background task, not awaited, for the same reason `start` gives.-- Guarded against overwriting a still-running task. The start-time run- spends most of its life in the agent read (`_create_agent_identities`- awaits `get_all` before the per-agent loop), so an attach during that- window must not spawn a second provisioner: two concurrent+ """Run identity provisioning when a shared connection attaches.++ A shared bridge started before the connection was up skipped the+ start-time run, as it had no connection yet. Attaching gives it one,+ so provision now — a fresh background task, not awaited, for the same+ reason `start` gives. `start` and `agents_changed` begin their runs+ here too, for the guard below.++ Guarded against overwriting a still-running task. A run spends most+ of its life in the agent read (`_create_agent_identities` awaits+ `get_all` before the per-agent loop), and an attach can land while the+ adapter is starting, before `start` asks for its own run, so a second+ call during a run must not spawn a second provisioner: two concurrent`create_agent_identity` calls can double a role through itscheck-then-create window, and overwriting the handle would orphan the- task `stop` cancels. If the start-time task is still going it will reach- the per-agent loop against the now-attached connection and provision- itself; if it has finished — the usual case, since the per-agent loop- fails fast while unattached — this replaces the completed handle.+ task `stop` cancels. A run still going reaches the per-agent loop+ against the connection as it is by then; one that has finished is+ replaced."""if self._identity_task is not None and not self._identity_task.done():returndiff --git a/core/switch_core/bridges/collaboration/discord/adapter.py b/core/switch_core/bridges/collaboration/discord/adapter.pyindex 86869004b..214a71ed7 100644--- a/core/switch_core/bridges/collaboration/discord/adapter.py+++ b/core/switch_core/bridges/collaboration/discord/adapter.py@@ -749,6 +749,9 @@ class DiscordAdapter(PlatformAdapter):def set_on_attached(self, callback: Callable[[], None]) -> None:self._on_attached = callback+ def awaiting_shared_connection(self) -> bool:+ return not self._owns_connection and self._connection is None+def attach_shared_connection(self, connection: DiscordConnection) -> None:"""Attach the shared Gateway connection to an inert shared bridge.diff --git a/core/switch_core/bridges/collaboration/discord/gateway.py b/core/switch_core/bridges/collaboration/discord/gateway.pyindex 4eb57c2d0..78584f5a5 100644--- a/core/switch_core/bridges/collaboration/discord/gateway.py+++ b/core/switch_core/bridges/collaboration/discord/gateway.py@@ -161,15 +161,19 @@ class DiscordGatewayClient:await self._bring_up(listen=True)async def _bring_up(self, *, listen: bool) -> None:- """Open the socket, or log in for the API alone, retrying with backoff- until it succeeds, then attach the bridges.+ """Open the socket, or log in for the API alone, and attach the+ bridges, retrying the whole with backoff until it succeeds.Run in the background so a configured-but-unreachable Discord appnever blocks or fails boot, or a take-over. Only the *initial* connectis retried here — once it is up, discord.py reconnects transient dropson its own (the same connection object, so attached bridges keep- working). On success `on_connected` fires, which walks the running- bridges and attaches the ones on a shared connection.+ working). Once connected, `on_connected` walks the running bridges and+ attaches the ones on a shared connection. A walk that raises is a+ failed bring-up like a connect that does: the connection is closed and+ the whole is tried again. Otherwise a process holding the socket would+ keep it open with no bridge attached, and Discord would go silent+ everywhere until the socket next moved."""delay = _INITIAL_RETRY_DELAYwhile True:@@ -178,7 +182,12 @@ class DiscordGatewayClient:await self.start()else:await self._connection.login()- break+ # Before the walk, with no await between: a bridge starting+ # from here on is attached as it starts, and every one started+ # before is in the walk, so none falls between the two.+ self._live = True+ await self._on_connected(self._connection)+ returnexcept asyncio.CancelledError:raiseexcept Exception:@@ -188,14 +197,10 @@ class DiscordGatewayClient:"Gateway connection" if listen else "API log-in",delay,)+ self._live = Falseawait self._connection.close()await asyncio.sleep(delay)delay = min(delay * 2, _MAX_RETRY_DELAY)- # Before the walk, with no await between: a bridge starting from here- # on is attached as it starts, and every one started before is in the- # walk, so none falls between the two.- self._live = True- await self._on_connected(self._connection)def own(self, ownership: Ownership) -> None:"""Make the socket owned work: opened in the one process that holds@@ -238,9 +243,8 @@ class DiscordGatewayClient:async def _close(self) -> None:"""Stop whatever is being brought up, and close the connection.- Closed even when the bring-up failed after connecting, in- `on_connected`, and its error is raised from here: the ownership logs- a hand-over that raises and releases the socket's lease all the same,+ Closed even when stopping the bring-up raises: the ownership logs a+ hand-over that raises and releases the socket's lease all the same,and the next holder opens a socket of its own."""task, self._task = self._task, Nonediff --git a/core/switch_core/bridges/collaboration/teams/adapter.py b/core/switch_core/bridges/collaboration/teams/adapter.pyindex 91b0a50a7..9d5bb1886 100644--- a/core/switch_core/bridges/collaboration/teams/adapter.py+++ b/core/switch_core/bridges/collaboration/teams/adapter.py@@ -4226,7 +4226,8 @@ class TeamsAdapter(PlatformAdapter):the distributed app the deployment's route has already checked Graph'svalidation tokens and routed the item by the organisation they vouchfor; what is checked here holds either way — the organisation's own- `clientState`, and that the subscription is one this bridge made.+ `clientState`, and that the subscription is one this bridge made (on+ a quiet copy, a removal is left for the holder to check)."""# Authenticate origin BEFORE acting on anything — data notifications and# lifecycle events alike. clientState is the bring-your-own bridge's only@@ -4243,7 +4244,20 @@ class TeamsAdapter(PlatformAdapter):logger.warning("Rejected Graph notification: clientState mismatch")returnsubscription_id = str(item.get("subscriptionId", ""))- if identity.shared and not await self._knows_subscription(subscription_id):+ # A quiet copy relays a removal to the holder without first knowing+ # the subscription. It adopts none at start, and Graph no longer lists+ # one it has removed, so reading them again cannot find it either: the+ # check would refuse every removal that reached this process, and the+ # holder would leave capture off until its repair loop noticed. The+ # holder recreates only a subscription it holds, so it decides.+ relays_removal = (+ not self._listening and item.get("lifecycleEvent") == "subscriptionRemoved"+ )+ if (+ identity.shared+ and not relays_removal+ and not await self._knows_subscription(subscription_id)+ ):logger.warning("Rejected a Graph notification for subscription %s, which the ""bridge for organisation %s did not make%s",diff --git a/core/switch_core/bridges/collaboration/telegram/adapter.py b/core/switch_core/bridges/collaboration/telegram/adapter.pyindex 407e7a64f..4e7288d9f 100644--- a/core/switch_core/bridges/collaboration/telegram/adapter.py+++ b/core/switch_core/bridges/collaboration/telegram/adapter.py@@ -1050,6 +1050,9 @@ class TelegramAdapter(PlatformAdapter):def set_on_attached(self, callback: Callable[[], None]) -> None:self._on_attached = callback+ def awaiting_shared_connection(self) -> bool:+ return self._shared and self._bot is None+def attach_shared_connection(self, connection: Any) -> None:"""Hand a shared bridge the deployment's bot, once.diff --git a/core/tests/switch_core/bridges/collaboration/test_bridge_channel_migration.py b/core/tests/switch_core/bridges/collaboration/test_bridge_channel_migration.pyindex 3a00c889b..8836b716a 100644--- a/core/tests/switch_core/bridges/collaboration/test_bridge_channel_migration.py+++ b/core/tests/switch_core/bridges/collaboration/test_bridge_channel_migration.py@@ -111,6 +111,7 @@ def test_the_handler_is_installed_before_the_adapter_starts() -> None:bridge._ensure_channel_captures = _noop # type: ignore[assignment,method-assign]bridge._refresh_channel_types = _noop # type: ignore[assignment,method-assign]bridge._create_agent_identities = _noop # type: ignore[assignment,method-assign]+ bridge._identity_task = Noneasyncio.run(CollaborationCore.start(bridge))diff --git a/core/tests/switch_core/bridges/collaboration/test_bridge_identity_backfill.py b/core/tests/switch_core/bridges/collaboration/test_bridge_identity_backfill.pyindex 3da8db100..34334b753 100644--- a/core/tests/switch_core/bridges/collaboration/test_bridge_identity_backfill.py+++ b/core/tests/switch_core/bridges/collaboration/test_bridge_identity_backfill.py@@ -193,8 +193,8 @@ async def test_attach_reprovision_does_not_overwrite_a_running_task() -> None:async def test_attach_reprovision_runs_when_no_task_is_live() -> None:- """The usual case: the start-time run has finished (it fails fast while the- bridge is unattached), so attaching spawns a fresh provisioning run."""+ """The usual case: no run is going (a shared bridge skips the start-time+ run while unattached), so attaching spawns a fresh provisioning run."""runs = 0done = asyncio.Event()@@ -212,3 +212,49 @@ async def test_attach_reprovision_runs_when_no_task_is_live() -> None:assert runs == 1assert core._identity_task is not Noneawait core.stop()+++class _SharedAdapter(_FakeAdapter):+ """A bridge on its platform's shared connection, attached after it starts."""++ def __init__(self) -> None:+ super().__init__()+ self.connection: object | None = None+ self._on_attached: Any = None++ def set_on_attached(self, callback: Any) -> None:+ self._on_attached = callback++ def attach_shared_connection(self, connection: object) -> None:+ self.connection = connection+ if self._on_attached is not None:+ self._on_attached()++ def awaiting_shared_connection(self) -> bool:+ return self.connection is None+++async def test_a_shared_bridge_provisions_once_attached_rather_than_at_start() -> None:+ """Started before the shared connection is up, every agent's call would+ fail, each logged, and attaching provisions them all anyway."""+ runs = 0+ done = asyncio.Event()++ async def _provision() -> None:+ nonlocal runs+ runs += 1+ done.set()++ core, _ = _core(_provision)+ adapter = _SharedAdapter()+ core._adapter = adapter # type: ignore[assignment]++ await core.start()+ await asyncio.sleep(0)+ assert core._identity_task is None+ assert runs == 0++ adapter.attach_shared_connection(object())+ await asyncio.wait_for(done.wait(), timeout=1)+ assert runs == 1+ await core.stop()diff --git a/core/tests/switch_core/bridges/collaboration/test_discord_gateway_ownership.py b/core/tests/switch_core/bridges/collaboration/test_discord_gateway_ownership.pyindex 75811d33f..3f88ccce5 100644--- a/core/tests/switch_core/bridges/collaboration/test_discord_gateway_ownership.py+++ b/core/tests/switch_core/bridges/collaboration/test_discord_gateway_ownership.py@@ -24,6 +24,7 @@ import pytestfrom sqlalchemy import func, updatefrom sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker+from switch_core.bridges.collaboration.discord import gateway as gateway_modulefrom switch_core.bridges.collaboration.discord.adapter import (DiscordAdapter,DiscordConnectionConfig,@@ -93,6 +94,8 @@ class _WithGateway:process: _Processgateway: DiscordGatewayClientconnection: _Connection+ # How many times the gateway has tried to attach the bridges.+ attach_attempts: Callable[[], int]async def boot(self) -> None:"""Start as `main` does, the gateway wanted before the first balance@@ -116,20 +119,32 @@ class _WithGateway:def holds_every(self, bridge_ids: list[str]) -> bool:return all(self.process.bridges.holds(b) for b in bridge_ids)+ def attached_every(self, bridge_ids: list[str]) -> bool:+ return all(+ getattr(self.process.bridges.get_adapter(b), "_connection", None)+ is self.connection+ for b in bridge_ids+ )+async def _with_gateway(postgres_url: str,session_factory: async_sessionmaker[AsyncSession],discord: _Discord,*,- attach_fails: bool = False,+ attach_failures: int = 0,) -> _WithGateway:+ """`attach_failures` is how many times attaching the bridges fails before+ it succeeds."""process = await _bridge_process(postgres_url, session_factory)process.bridges.register_adapter("discord", DiscordAdapter, DiscordConnectionConfig)bridges = process.bridges+ attempts = 0async def attach(connection: DiscordConnection) -> None:- if attach_fails:+ nonlocal attempts+ attempts += 1+ if attempts <= attach_failures:raise RuntimeError("attaching the bridges failed")for adapter in bridges.iter_adapters():if isinstance(adapter, DiscordAdapter):@@ -145,7 +160,12 @@ async def _with_gateway(connection = _Connection(discord)gateway._connection = connection # type: ignore[assignment]bridges.add_bridge_starting_listener(gateway.attach_if_live)- return _WithGateway(process=process, gateway=gateway, connection=connection)+ return _WithGateway(+ process=process,+ gateway=gateway,+ connection=connection,+ attach_attempts=lambda: attempts,+ )@pytest.fixture@@ -242,7 +262,7 @@ class TestOneSocket:assert not any(other.process.bridges.holds(b) for b in shared_bridges)assert owner.connection.discord.most_sockets == 1- async def test_every_copy_of_a_bridge_is_attached_to_its_own_processs_connection(+ async def test_every_copy_of_a_bridge_is_attached_to_its_own_process_connection(self,two: tuple[_WithGateway, _WithGateway],shared_bridges: list[str],@@ -258,6 +278,34 @@ class TestOneSocket:)+class TestBringingUp:+ async def test_attaching_the_bridges_is_retried_when_it_fails(+ self,+ postgres_url: str,+ session_factory: async_sessionmaker[AsyncSession],+ shared_bridges: list[str],+ monkeypatch: pytest.MonkeyPatch,+ ) -> None:+ """A process that takes the socket and fails to attach its bridges+ tries the bring-up again, rather than holding the socket with no+ bridge attached, which would leave Discord silent until the socket+ next moved."""+ monkeypatch.setattr(gateway_module, "_INITIAL_RETRY_DELAY", 0.05)+ discord = _Discord()+ process = await _with_gateway(+ postgres_url, session_factory, discord, attach_failures=1+ )+ try:+ await process.boot()++ await _eventually(lambda: process.attached_every(shared_bridges))+ assert process.attach_attempts() == 2+ assert process.holds_every(shared_bridges)+ assert discord.sockets == [process.connection]+ finally:+ await process.stop()++class TestHandingOver:async def test_a_process_stopping_hands_the_socket_and_its_bridges_over(self,@@ -276,26 +324,28 @@ class TestHandingOver:assert other.connection.discord.sockets == [other.connection]assert other.connection.discord.most_sockets == 1- async def test_a_socket_whose_bring_up_failed_is_closed_as_it_is_handed_over(+ async def test_a_socket_whose_bring_up_keeps_failing_is_closed_as_it_is_handed_over(self,postgres_url: str,session_factory: async_sessionmaker[AsyncSession],shared_bridges: list[str],+ monkeypatch: pytest.MonkeyPatch,) -> None:- """Attaching the bridges failed once the socket was up, and handing- the socket over raises that failure. The ownership releases the- socket's lease all the same, and the next holder opens a socket of its- own, so this one has to be closed by then."""+ """Attaching the bridges keeps failing once the socket is up, so the+ bring-up is still being retried when the socket is handed over. The+ hand-over stops it, and the next holder opens a socket of its own, so+ this one has to be closed by then."""+ monkeypatch.setattr(gateway_module, "_INITIAL_RETRY_DELAY", 0.05)discord = _Discord()started: list[_WithGateway] = []try:failing = await _with_gateway(- postgres_url, session_factory, discord, attach_fails=True+ postgres_url, session_factory, discord, attach_failures=1_000)started.append(failing)# Alone, so it takes the socket.await failing.boot()- await _eventually(failing.has_socket)+ await _eventually(lambda: failing.attach_attempts() >= 2)other = await _with_gateway(postgres_url, session_factory, discord)started.append(other)await other.boot()diff --git a/core/tests/switch_core/bridges/collaboration/test_install_claim.py b/core/tests/switch_core/bridges/collaboration/test_install_claim.pyindex 3daae17d4..4213cdbfc 100644--- a/core/tests/switch_core/bridges/collaboration/test_install_claim.py+++ b/core/tests/switch_core/bridges/collaboration/test_install_claim.py@@ -248,6 +248,9 @@ class _AttachableAdapter(_RecordingAdapter):def set_on_attached(self, callback: Any) -> None:return None+ def awaiting_shared_connection(self) -> bool:+ return not self.attached+class _FakeLifecycle:"""Registers real rows, on a plain session like the real lifecycle.diff --git a/core/tests/switch_core/bridges/collaboration/test_teams_shared_mode.py b/core/tests/switch_core/bridges/collaboration/test_teams_shared_mode.pyindex 5ef8785c5..e2ac7ccf5 100644--- a/core/tests/switch_core/bridges/collaboration/test_teams_shared_mode.py+++ b/core/tests/switch_core/bridges/collaboration/test_teams_shared_mode.py@@ -1648,6 +1648,43 @@ async def test_a_notification_before_the_subscriptions_could_be_read_reads_them(assert adapter._subscriptions == {CHANNEL: "LIVE"}+async def test_a_quiet_copy_relays_a_removal_it_cannot_know_to_the_holder() -> None:+ """A quiet copy adopts no subscriptions at start, and Graph no longer+ lists one it has removed, so the copy cannot know it. The holder can,+ and recreates it; a removal stopped at the copy would leave capture off+ until the holder's repair loop noticed. The organisation's clientState+ is still checked before anything is relayed."""+ adapter = _shared_adapter(team_id="team-1")+ adapter.set_listening(False)+ relayed: list[dict[str, str]] = []+ adapter.set_holder_relay(relayed.append)+ graph = _SubscribingGraph(list_failures=0, held=[])+ adapter._graph = graph # type: ignore[assignment]++ await adapter.receive_notification(+ {+ "lifecycleEvent": "subscriptionRemoved",+ "subscriptionId": "REMOVED",+ "clientState": client_state_for("jwt-secret", OTHER_ORG),+ }+ )+ await adapter.receive_notification(+ {+ "lifecycleEvent": "subscriptionRemoved",+ "subscriptionId": "REMOVED",+ "clientState": adapter._me.client_state,+ }+ )++ assert relayed == [+ {+ "action": "lifecycle",+ "event": "subscriptionRemoved",+ "subscription_id": "REMOVED",+ }+ ]++# ── Capture only where Switch is ─────────────────────────────────────────────