Skip to content

Open-source AI teammates in Slack, Teams, Discord, Telegram and Mattermost. Self-host it, with any agent harness and any model.

Retry attaching Discord bridges, relay Teams subscription removals, and stop the Discord identity errors at boot (#890)

Abel Dantas3bd3b801f688
diff --git a/core/switch_core/bridges/collaboration/adapter.py b/core/switch_core/bridges/collaboration/adapter.py
index 098436ba1..c70d98b08 100644
--- a/core/switch_core/bridges/collaboration/adapter.py
+++ b/core/switch_core/bridges/collaboration/adapter.py
@@ -393,6 +393,11 @@ class SupportsSharedConnection(Protocol):
def set_on_attached(self, callback: Callable[[], None]) -> None: ...
+ def awaiting_shared_connection(self) -> bool:
+ """Whether this bridge runs on the shared connection and has not been
+ attached to it yet, so nothing that reaches the platform can run."""
+ ...
+
class PlatformAdapter(ABC):
#: The platform as a person names it — "Microsoft Teams", not "teams".
diff --git a/core/switch_core/bridges/collaboration/collaboration_core.py b/core/switch_core/bridges/collaboration/collaboration_core.py
index d2e93de5f..e5625c15a 100644
--- a/core/switch_core/bridges/collaboration/collaboration_core.py
+++ b/core/switch_core/bridges/collaboration/collaboration_core.py
@@ -531,7 +531,11 @@ class CollaborationCore:
# restarts one. Messages do not depend on it: an agent is addressable
# by name whether or not its platform identity exists yet. Every
# process provisions, the adapter deciding how much a quiet one needs.
- self._identity_task = asyncio.create_task(self._run_agent_identities())
+ # Not yet on a shared connection that is not attached: every agent's
+ # call would fail, and attaching provisions them all. Guarded, since
+ # an attach while the adapter started may have begun a run already.
+ if not self._awaiting_shared_connection():
+ self._provision_identities_on_attach()
# Not awaited either: one platform read per channel, and nothing waits
# on it. Only where the bridge is held: one process is enough.
if self._active:
@@ -550,7 +554,16 @@ class CollaborationCore:
let go of the deleted ones'. A run already going reads the agents
again once it is done, in case it read them before the change."""
self._identities_stale = True
- self._provision_identities_on_attach()
+ if not self._awaiting_shared_connection():
+ self._provision_identities_on_attach()
+
+ def _awaiting_shared_connection(self) -> bool:
+ """Whether the bridge waits to be attached to its platform's shared
+ connection, which provisions the agents' identities once it is."""
+ return (
+ isinstance(self._adapter, SupportsSharedConnection)
+ and self._adapter.awaiting_shared_connection()
+ )
async def stop(self) -> None:
try:
@@ -646,22 +659,24 @@ class CollaborationCore:
self._human_user_ids.add(client.transport_user_id)
def _provision_identities_on_attach(self) -> None:
- """Re-run identity provisioning when a shared connection attaches.
-
- The start-time run failed for a shared bridge because it had no
- connection yet. Attaching gives it one, so provision now — a fresh
- background task, not awaited, for the same reason `start` gives.
-
- Guarded against overwriting a still-running task. The start-time run
- spends most of its life in the agent read (`_create_agent_identities`
- awaits `get_all` before the per-agent loop), so an attach during that
- window must not spawn a second provisioner: two concurrent
+ """Run identity provisioning when a shared connection attaches.
+
+ A shared bridge started before the connection was up skipped the
+ start-time run, as it had no connection yet. Attaching gives it one,
+ so provision now — a fresh background task, not awaited, for the same
+ reason `start` gives. `start` and `agents_changed` begin their runs
+ here too, for the guard below.
+
+ Guarded against overwriting a still-running task. A run spends most
+ of its life in the agent read (`_create_agent_identities` awaits
+ `get_all` before the per-agent loop), and an attach can land while the
+ adapter is starting, before `start` asks for its own run, so a second
+ call during a run must not spawn a second provisioner: two concurrent
`create_agent_identity` calls can double a role through its
check-then-create window, and overwriting the handle would orphan the
- task `stop` cancels. If the start-time task is still going it will reach
- the per-agent loop against the now-attached connection and provision
- itself; if it has finished — the usual case, since the per-agent loop
- fails fast while unattached — this replaces the completed handle.
+ task `stop` cancels. A run still going reaches the per-agent loop
+ against the connection as it is by then; one that has finished is
+ replaced.
"""
if self._identity_task is not None and not self._identity_task.done():
return
diff --git a/core/switch_core/bridges/collaboration/discord/adapter.py b/core/switch_core/bridges/collaboration/discord/adapter.py
index 86869004b..214a71ed7 100644
--- a/core/switch_core/bridges/collaboration/discord/adapter.py
+++ b/core/switch_core/bridges/collaboration/discord/adapter.py
@@ -749,6 +749,9 @@ class DiscordAdapter(PlatformAdapter):
def set_on_attached(self, callback: Callable[[], None]) -> None:
self._on_attached = callback
+ def awaiting_shared_connection(self) -> bool:
+ return not self._owns_connection and self._connection is None
+
def attach_shared_connection(self, connection: DiscordConnection) -> None:
"""Attach the shared Gateway connection to an inert shared bridge.
diff --git a/core/switch_core/bridges/collaboration/discord/gateway.py b/core/switch_core/bridges/collaboration/discord/gateway.py
index 4eb57c2d0..78584f5a5 100644
--- a/core/switch_core/bridges/collaboration/discord/gateway.py
+++ b/core/switch_core/bridges/collaboration/discord/gateway.py
@@ -161,15 +161,19 @@ class DiscordGatewayClient:
await self._bring_up(listen=True)
async def _bring_up(self, *, listen: bool) -> None:
- """Open the socket, or log in for the API alone, retrying with backoff
- until it succeeds, then attach the bridges.
+ """Open the socket, or log in for the API alone, and attach the
+ bridges, retrying the whole with backoff until it succeeds.
Run in the background so a configured-but-unreachable Discord app
never blocks or fails boot, or a take-over. Only the *initial* connect
is retried here — once it is up, discord.py reconnects transient drops
on its own (the same connection object, so attached bridges keep
- working). On success `on_connected` fires, which walks the running
- bridges and attaches the ones on a shared connection.
+ working). Once connected, `on_connected` walks the running bridges and
+ attaches the ones on a shared connection. A walk that raises is a
+ failed bring-up like a connect that does: the connection is closed and
+ the whole is tried again. Otherwise a process holding the socket would
+ keep it open with no bridge attached, and Discord would go silent
+ everywhere until the socket next moved.
"""
delay = _INITIAL_RETRY_DELAY
while True:
@@ -178,7 +182,12 @@ class DiscordGatewayClient:
await self.start()
else:
await self._connection.login()
- break
+ # Before the walk, with no await between: a bridge starting
+ # from here on is attached as it starts, and every one started
+ # before is in the walk, so none falls between the two.
+ self._live = True
+ await self._on_connected(self._connection)
+ return
except asyncio.CancelledError:
raise
except Exception:
@@ -188,14 +197,10 @@ class DiscordGatewayClient:
"Gateway connection" if listen else "API log-in",
delay,
)
+ self._live = False
await self._connection.close()
await asyncio.sleep(delay)
delay = min(delay * 2, _MAX_RETRY_DELAY)
- # Before the walk, with no await between: a bridge starting from here
- # on is attached as it starts, and every one started before is in the
- # walk, so none falls between the two.
- self._live = True
- await self._on_connected(self._connection)
def own(self, ownership: Ownership) -> None:
"""Make the socket owned work: opened in the one process that holds
@@ -238,9 +243,8 @@ class DiscordGatewayClient:
async def _close(self) -> None:
"""Stop whatever is being brought up, and close the connection.
- Closed even when the bring-up failed after connecting, in
- `on_connected`, and its error is raised from here: the ownership logs
- a hand-over that raises and releases the socket's lease all the same,
+ Closed even when stopping the bring-up raises: the ownership logs a
+ hand-over that raises and releases the socket's lease all the same,
and the next holder opens a socket of its own.
"""
task, self._task = self._task, None
diff --git a/core/switch_core/bridges/collaboration/teams/adapter.py b/core/switch_core/bridges/collaboration/teams/adapter.py
index 91b0a50a7..9d5bb1886 100644
--- a/core/switch_core/bridges/collaboration/teams/adapter.py
+++ b/core/switch_core/bridges/collaboration/teams/adapter.py
@@ -4226,7 +4226,8 @@ class TeamsAdapter(PlatformAdapter):
the distributed app the deployment's route has already checked Graph's
validation tokens and routed the item by the organisation they vouch
for; what is checked here holds either way — the organisation's own
- `clientState`, and that the subscription is one this bridge made.
+ `clientState`, and that the subscription is one this bridge made (on
+ a quiet copy, a removal is left for the holder to check).
"""
# Authenticate origin BEFORE acting on anything — data notifications and
# lifecycle events alike. clientState is the bring-your-own bridge's only
@@ -4243,7 +4244,20 @@ class TeamsAdapter(PlatformAdapter):
logger.warning("Rejected Graph notification: clientState mismatch")
return
subscription_id = str(item.get("subscriptionId", ""))
- if identity.shared and not await self._knows_subscription(subscription_id):
+ # A quiet copy relays a removal to the holder without first knowing
+ # the subscription. It adopts none at start, and Graph no longer lists
+ # one it has removed, so reading them again cannot find it either: the
+ # check would refuse every removal that reached this process, and the
+ # holder would leave capture off until its repair loop noticed. The
+ # holder recreates only a subscription it holds, so it decides.
+ relays_removal = (
+ not self._listening and item.get("lifecycleEvent") == "subscriptionRemoved"
+ )
+ if (
+ identity.shared
+ and not relays_removal
+ and not await self._knows_subscription(subscription_id)
+ ):
logger.warning(
"Rejected a Graph notification for subscription %s, which the "
"bridge for organisation %s did not make%s",
diff --git a/core/switch_core/bridges/collaboration/telegram/adapter.py b/core/switch_core/bridges/collaboration/telegram/adapter.py
index 407e7a64f..4e7288d9f 100644
--- a/core/switch_core/bridges/collaboration/telegram/adapter.py
+++ b/core/switch_core/bridges/collaboration/telegram/adapter.py
@@ -1050,6 +1050,9 @@ class TelegramAdapter(PlatformAdapter):
def set_on_attached(self, callback: Callable[[], None]) -> None:
self._on_attached = callback
+ def awaiting_shared_connection(self) -> bool:
+ return self._shared and self._bot is None
+
def attach_shared_connection(self, connection: Any) -> None:
"""Hand a shared bridge the deployment's bot, once.
diff --git a/core/tests/switch_core/bridges/collaboration/test_bridge_channel_migration.py b/core/tests/switch_core/bridges/collaboration/test_bridge_channel_migration.py
index 3a00c889b..8836b716a 100644
--- a/core/tests/switch_core/bridges/collaboration/test_bridge_channel_migration.py
+++ b/core/tests/switch_core/bridges/collaboration/test_bridge_channel_migration.py
@@ -111,6 +111,7 @@ def test_the_handler_is_installed_before_the_adapter_starts() -> None:
bridge._ensure_channel_captures = _noop # type: ignore[assignment,method-assign]
bridge._refresh_channel_types = _noop # type: ignore[assignment,method-assign]
bridge._create_agent_identities = _noop # type: ignore[assignment,method-assign]
+ bridge._identity_task = None
asyncio.run(CollaborationCore.start(bridge))
diff --git a/core/tests/switch_core/bridges/collaboration/test_bridge_identity_backfill.py b/core/tests/switch_core/bridges/collaboration/test_bridge_identity_backfill.py
index 3da8db100..34334b753 100644
--- a/core/tests/switch_core/bridges/collaboration/test_bridge_identity_backfill.py
+++ b/core/tests/switch_core/bridges/collaboration/test_bridge_identity_backfill.py
@@ -193,8 +193,8 @@ async def test_attach_reprovision_does_not_overwrite_a_running_task() -> None:
async def test_attach_reprovision_runs_when_no_task_is_live() -> None:
- """The usual case: the start-time run has finished (it fails fast while the
- bridge is unattached), so attaching spawns a fresh provisioning run."""
+ """The usual case: no run is going (a shared bridge skips the start-time
+ run while unattached), so attaching spawns a fresh provisioning run."""
runs = 0
done = asyncio.Event()
@@ -212,3 +212,49 @@ async def test_attach_reprovision_runs_when_no_task_is_live() -> None:
assert runs == 1
assert core._identity_task is not None
await core.stop()
+
+
+class _SharedAdapter(_FakeAdapter):
+ """A bridge on its platform's shared connection, attached after it starts."""
+
+ def __init__(self) -> None:
+ super().__init__()
+ self.connection: object | None = None
+ self._on_attached: Any = None
+
+ def set_on_attached(self, callback: Any) -> None:
+ self._on_attached = callback
+
+ def attach_shared_connection(self, connection: object) -> None:
+ self.connection = connection
+ if self._on_attached is not None:
+ self._on_attached()
+
+ def awaiting_shared_connection(self) -> bool:
+ return self.connection is None
+
+
+async def test_a_shared_bridge_provisions_once_attached_rather_than_at_start() -> None:
+ """Started before the shared connection is up, every agent's call would
+ fail, each logged, and attaching provisions them all anyway."""
+ runs = 0
+ done = asyncio.Event()
+
+ async def _provision() -> None:
+ nonlocal runs
+ runs += 1
+ done.set()
+
+ core, _ = _core(_provision)
+ adapter = _SharedAdapter()
+ core._adapter = adapter # type: ignore[assignment]
+
+ await core.start()
+ await asyncio.sleep(0)
+ assert core._identity_task is None
+ assert runs == 0
+
+ adapter.attach_shared_connection(object())
+ await asyncio.wait_for(done.wait(), timeout=1)
+ assert runs == 1
+ await core.stop()
diff --git a/core/tests/switch_core/bridges/collaboration/test_discord_gateway_ownership.py b/core/tests/switch_core/bridges/collaboration/test_discord_gateway_ownership.py
index 75811d33f..3f88ccce5 100644
--- a/core/tests/switch_core/bridges/collaboration/test_discord_gateway_ownership.py
+++ b/core/tests/switch_core/bridges/collaboration/test_discord_gateway_ownership.py
@@ -24,6 +24,7 @@ import pytest
from sqlalchemy import func, update
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
+from switch_core.bridges.collaboration.discord import gateway as gateway_module
from switch_core.bridges.collaboration.discord.adapter import (
DiscordAdapter,
DiscordConnectionConfig,
@@ -93,6 +94,8 @@ class _WithGateway:
process: _Process
gateway: DiscordGatewayClient
connection: _Connection
+ # How many times the gateway has tried to attach the bridges.
+ attach_attempts: Callable[[], int]
async def boot(self) -> None:
"""Start as `main` does, the gateway wanted before the first balance
@@ -116,20 +119,32 @@ class _WithGateway:
def holds_every(self, bridge_ids: list[str]) -> bool:
return all(self.process.bridges.holds(b) for b in bridge_ids)
+ def attached_every(self, bridge_ids: list[str]) -> bool:
+ return all(
+ getattr(self.process.bridges.get_adapter(b), "_connection", None)
+ is self.connection
+ for b in bridge_ids
+ )
+
async def _with_gateway(
postgres_url: str,
session_factory: async_sessionmaker[AsyncSession],
discord: _Discord,
*,
- attach_fails: bool = False,
+ attach_failures: int = 0,
) -> _WithGateway:
+ """`attach_failures` is how many times attaching the bridges fails before
+ it succeeds."""
process = await _bridge_process(postgres_url, session_factory)
process.bridges.register_adapter("discord", DiscordAdapter, DiscordConnectionConfig)
bridges = process.bridges
+ attempts = 0
async def attach(connection: DiscordConnection) -> None:
- if attach_fails:
+ nonlocal attempts
+ attempts += 1
+ if attempts <= attach_failures:
raise RuntimeError("attaching the bridges failed")
for adapter in bridges.iter_adapters():
if isinstance(adapter, DiscordAdapter):
@@ -145,7 +160,12 @@ async def _with_gateway(
connection = _Connection(discord)
gateway._connection = connection # type: ignore[assignment]
bridges.add_bridge_starting_listener(gateway.attach_if_live)
- return _WithGateway(process=process, gateway=gateway, connection=connection)
+ return _WithGateway(
+ process=process,
+ gateway=gateway,
+ connection=connection,
+ attach_attempts=lambda: attempts,
+ )
@pytest.fixture
@@ -242,7 +262,7 @@ class TestOneSocket:
assert not any(other.process.bridges.holds(b) for b in shared_bridges)
assert owner.connection.discord.most_sockets == 1
- async def test_every_copy_of_a_bridge_is_attached_to_its_own_processs_connection(
+ async def test_every_copy_of_a_bridge_is_attached_to_its_own_process_connection(
self,
two: tuple[_WithGateway, _WithGateway],
shared_bridges: list[str],
@@ -258,6 +278,34 @@ class TestOneSocket:
)
+class TestBringingUp:
+ async def test_attaching_the_bridges_is_retried_when_it_fails(
+ self,
+ postgres_url: str,
+ session_factory: async_sessionmaker[AsyncSession],
+ shared_bridges: list[str],
+ monkeypatch: pytest.MonkeyPatch,
+ ) -> None:
+ """A process that takes the socket and fails to attach its bridges
+ tries the bring-up again, rather than holding the socket with no
+ bridge attached, which would leave Discord silent until the socket
+ next moved."""
+ monkeypatch.setattr(gateway_module, "_INITIAL_RETRY_DELAY", 0.05)
+ discord = _Discord()
+ process = await _with_gateway(
+ postgres_url, session_factory, discord, attach_failures=1
+ )
+ try:
+ await process.boot()
+
+ await _eventually(lambda: process.attached_every(shared_bridges))
+ assert process.attach_attempts() == 2
+ assert process.holds_every(shared_bridges)
+ assert discord.sockets == [process.connection]
+ finally:
+ await process.stop()
+
+
class TestHandingOver:
async def test_a_process_stopping_hands_the_socket_and_its_bridges_over(
self,
@@ -276,26 +324,28 @@ class TestHandingOver:
assert other.connection.discord.sockets == [other.connection]
assert other.connection.discord.most_sockets == 1
- async def test_a_socket_whose_bring_up_failed_is_closed_as_it_is_handed_over(
+ async def test_a_socket_whose_bring_up_keeps_failing_is_closed_as_it_is_handed_over(
self,
postgres_url: str,
session_factory: async_sessionmaker[AsyncSession],
shared_bridges: list[str],
+ monkeypatch: pytest.MonkeyPatch,
) -> None:
- """Attaching the bridges failed once the socket was up, and handing
- the socket over raises that failure. The ownership releases the
- socket's lease all the same, and the next holder opens a socket of its
- own, so this one has to be closed by then."""
+ """Attaching the bridges keeps failing once the socket is up, so the
+ bring-up is still being retried when the socket is handed over. The
+ hand-over stops it, and the next holder opens a socket of its own, so
+ this one has to be closed by then."""
+ monkeypatch.setattr(gateway_module, "_INITIAL_RETRY_DELAY", 0.05)
discord = _Discord()
started: list[_WithGateway] = []
try:
failing = await _with_gateway(
- postgres_url, session_factory, discord, attach_fails=True
+ postgres_url, session_factory, discord, attach_failures=1_000
)
started.append(failing)
# Alone, so it takes the socket.
await failing.boot()
- await _eventually(failing.has_socket)
+ await _eventually(lambda: failing.attach_attempts() >= 2)
other = await _with_gateway(postgres_url, session_factory, discord)
started.append(other)
await other.boot()
diff --git a/core/tests/switch_core/bridges/collaboration/test_install_claim.py b/core/tests/switch_core/bridges/collaboration/test_install_claim.py
index 3daae17d4..4213cdbfc 100644
--- a/core/tests/switch_core/bridges/collaboration/test_install_claim.py
+++ b/core/tests/switch_core/bridges/collaboration/test_install_claim.py
@@ -248,6 +248,9 @@ class _AttachableAdapter(_RecordingAdapter):
def set_on_attached(self, callback: Any) -> None:
return None
+ def awaiting_shared_connection(self) -> bool:
+ return not self.attached
+
class _FakeLifecycle:
"""Registers real rows, on a plain session like the real lifecycle.
diff --git a/core/tests/switch_core/bridges/collaboration/test_teams_shared_mode.py b/core/tests/switch_core/bridges/collaboration/test_teams_shared_mode.py
index 5ef8785c5..e2ac7ccf5 100644
--- a/core/tests/switch_core/bridges/collaboration/test_teams_shared_mode.py
+++ b/core/tests/switch_core/bridges/collaboration/test_teams_shared_mode.py
@@ -1648,6 +1648,43 @@ async def test_a_notification_before_the_subscriptions_could_be_read_reads_them(
assert adapter._subscriptions == {CHANNEL: "LIVE"}
+async def test_a_quiet_copy_relays_a_removal_it_cannot_know_to_the_holder() -> None:
+ """A quiet copy adopts no subscriptions at start, and Graph no longer
+ lists one it has removed, so the copy cannot know it. The holder can,
+ and recreates it; a removal stopped at the copy would leave capture off
+ until the holder's repair loop noticed. The organisation's clientState
+ is still checked before anything is relayed."""
+ adapter = _shared_adapter(team_id="team-1")
+ adapter.set_listening(False)
+ relayed: list[dict[str, str]] = []
+ adapter.set_holder_relay(relayed.append)
+ graph = _SubscribingGraph(list_failures=0, held=[])
+ adapter._graph = graph # type: ignore[assignment]
+
+ await adapter.receive_notification(
+ {
+ "lifecycleEvent": "subscriptionRemoved",
+ "subscriptionId": "REMOVED",
+ "clientState": client_state_for("jwt-secret", OTHER_ORG),
+ }
+ )
+ await adapter.receive_notification(
+ {
+ "lifecycleEvent": "subscriptionRemoved",
+ "subscriptionId": "REMOVED",
+ "clientState": adapter._me.client_state,
+ }
+ )
+
+ assert relayed == [
+ {
+ "action": "lifecycle",
+ "event": "subscriptionRemoved",
+ "subscription_id": "REMOVED",
+ }
+ ]
+
+
# ── Capture only where Switch is ─────────────────────────────────────────────