Skip to content

Open-source AI teammates in Slack, Teams, Discord, Telegram and Mattermost. Self-host it, with any agent harness and any model.

fix(slack): stop a burst of rate limits from killing the agent card for good (#276)

Louis Amaudruz14e90db49339
diff --git a/core/switch_core/bridges/collaboration/slack/adapter.py b/core/switch_core/bridges/collaboration/slack/adapter.py
index 4d0825166..206a00cb2 100644
--- a/core/switch_core/bridges/collaboration/slack/adapter.py
+++ b/core/switch_core/bridges/collaboration/slack/adapter.py
@@ -92,6 +92,13 @@ def _group_description(agent_description: str) -> str:
# cannot work must not warn on every turn for the life of the bridge.
_AGENT_SESSIONS_FAILURE_LIMIT = 3
+# How long a give-up over unrecognised errors lasts before the bridge tries
+# again. Long enough that a bad patch is waited out rather than hammered
+# through, short enough that a bridge does not stay dark for a day because of
+# one. A refusal that named its own cause is not covered: nothing about the
+# workspace will have changed, so it is not retried at all.
+_AGENT_SESSIONS_RETRY_AFTER = 600
+
# Prefix on the trace lines that follow a turn through the session, so a run
# can be read end to end when something does not render. Debug level: the
# per-turn detail is only wanted when someone is looking for it.
@@ -265,10 +272,16 @@ class SlackAdapter(CollaborationAdapter):
# Set once Slack has told us it will not host agent sessions, so the
# bridge stops asking and says so only once.
self._agent_sessions_off_reason: str | None = None
+ # When a give-up over unrecognised errors expires. None means the
+ # reason above is a workspace's settled answer and will not improve on
+ # its own, so nothing re-arms it.
+ self._agent_sessions_retry_at: float | None = None
# Consecutive identical session-status failures, so an error code this
# build does not recognise still stops complaining eventually.
self._session_failures = 0
self._last_session_error: str | None = None
+ # While Slack is throttling us, when it said we may call again.
+ self._sessions_throttled_until: float | None = None
# (channel_id, thread_ts) -> agent whose turn owns that session, so the
# stop button can be routed to the agent it belongs to.
self._session_owner: dict[tuple[str, str], str] = {}
@@ -903,7 +916,7 @@ class SlackAdapter(CollaborationAdapter):
if not self._config.agent_sessions:
logger.debug(_TRACE + "skipped for %s: turned off in config", agent_name)
return
- if self._agent_sessions_off_reason:
+ if self._agent_sessions_off():
# Silent by design. Giving up is announced once, where it happens;
# saying so again per skipped turn produced eleven thousand lines
# in a night — an instrument that ruins the thing it measures.
@@ -928,6 +941,11 @@ class SlackAdapter(CollaborationAdapter):
self._threadless_logged.discard((channel_id, agent_name))
working = state in ("working", "awaiting-input")
+ if working and self._sessions_throttled():
+ # A step lost to throttling costs one line of the card. Teardown is
+ # not skipped the same way: dropping it would leave the card open
+ # for good, so it is attempted even while we are being throttled.
+ return
key = (channel_id, thread_ts)
if working:
self._session_owner[key] = agent_name
@@ -944,7 +962,12 @@ class SlackAdapter(CollaborationAdapter):
)
def _note_session_failure(
- self, error: str, agent_name: str, channel_id: str
+ self,
+ error: str,
+ agent_name: str,
+ channel_id: str,
+ *,
+ retry_after: int | None = None,
) -> None:
"""Log a failed status call, and give up if it is not going to improve.
@@ -954,7 +977,15 @@ class SlackAdapter(CollaborationAdapter):
list does not know about (`not_authorized` for an app that is not an
agent, found in the pilot), and without a backstop each one means a
warning on every turn for as long as the bridge runs.
+
+ Throttling is the exception, and is not counted at all: it says the app
+ is busy, not that it is unfit, and counting it turned a burst of
+ traffic into a bridge that never showed a card again.
"""
+ if error == "ratelimited":
+ self._note_session_throttled(retry_after or _RATE_LIMIT_DEFAULT_DELAY)
+ return
+
if error in _AGENT_SESSIONS_UNAVAILABLE_ERRORS:
self._disable_agent_sessions(error)
return
@@ -972,7 +1003,54 @@ class SlackAdapter(CollaborationAdapter):
error or "unknown error",
)
if self._session_failures >= _AGENT_SESSIONS_FAILURE_LIMIT:
- self._disable_agent_sessions(error)
+ self._disable_agent_sessions(error, retry_in=_AGENT_SESSIONS_RETRY_AFTER)
+
+ def _note_session_throttled(self, delay: int) -> None:
+ """Stand down for as long as Slack asked, and say so once per burst."""
+ already_waiting = self._sessions_throttled()
+ self._sessions_throttled_until = time.monotonic() + delay
+ if already_waiting:
+ logger.debug(_TRACE + "still throttled; waiting %ss more", delay)
+ return
+ logger.warning(
+ "Slack is rate-limiting agent session updates; pausing them for %ss. "
+ "Cards may miss a step until it clears; Switch's own status messages "
+ "are unaffected.",
+ delay,
+ )
+
+ def _sessions_throttled(self) -> bool:
+ until = self._sessions_throttled_until
+ if until is None:
+ return False
+ if time.monotonic() < until:
+ return True
+ self._sessions_throttled_until = None
+ logger.debug(_TRACE + "throttling window elapsed; resuming session updates")
+ return False
+
+ def _agent_sessions_off(self) -> bool:
+ """Whether sessions are given up on — re-arming a lapsed give-up.
+
+ A give-up over errors this build cannot name is a guess, so it expires:
+ the bridge tries again rather than staying dark until someone restarts
+ it. A refusal that named its own cause does not expire, because nothing
+ will have changed without an operator changing it.
+ """
+ if self._agent_sessions_off_reason is None:
+ return False
+ retry_at = self._agent_sessions_retry_at
+ if retry_at is None or time.monotonic() < retry_at:
+ return True
+ logger.info(
+ "Trying Slack agent sessions again after backing off over '%s'.",
+ self._agent_sessions_off_reason,
+ )
+ self._agent_sessions_off_reason = None
+ self._agent_sessions_retry_at = None
+ self._session_failures = 0
+ self._last_session_error = None
+ return False
async def _drive_stream(
self,
@@ -1099,29 +1177,45 @@ class SlackAdapter(CollaborationAdapter):
try:
result = await call()
except SlackApiError as e:
+ error = str(e.response.get("error", ""))
self._note_session_failure(
- str(e.response.get("error", "")), agent_name, channel_id
+ error,
+ agent_name,
+ channel_id,
+ retry_after=_retry_after_seconds(e) if error == "ratelimited" else None,
)
return None
self._session_failures = 0
+ self._sessions_throttled_until = None
return str(result.get("ts", ""))
- def _disable_agent_sessions(self, error: str) -> None:
+ def _disable_agent_sessions(
+ self, error: str, *, retry_in: int | None = None
+ ) -> None:
if self._agent_sessions_off_reason:
return
self._agent_sessions_off_reason = error
+ self._agent_sessions_retry_at = (
+ time.monotonic() + retry_in if retry_in is not None else None
+ )
reason = _AGENT_SESSIONS_UNAVAILABLE_ERRORS.get(
error,
f"Slack kept refusing with '{error or 'an unknown error'}'. If the "
"app is not declared as an Agent in its settings, that is the "
"likeliest cause.",
)
+ recovery = (
+ f" Trying again in {retry_in}s."
+ if retry_in is not None
+ else " This will not change without an operator changing it."
+ )
logger.warning(
"Slack agent sessions are unavailable for this app (%s): %s "
"Turns still show Switch's own status messages; only Slack's native "
- "loading UX and stop button are missing.",
+ "loading UX and stop button are missing.%s",
error,
reason,
+ recovery,
)
@staticmethod
diff --git a/core/tests/switch_core/bridges/collaboration/test_slack_agent_sessions.py b/core/tests/switch_core/bridges/collaboration/test_slack_agent_sessions.py
index 88c6344e0..73d04a1d5 100644
--- a/core/tests/switch_core/bridges/collaboration/test_slack_agent_sessions.py
+++ b/core/tests/switch_core/bridges/collaboration/test_slack_agent_sessions.py
@@ -9,6 +9,7 @@ matters here is which of the two appears, and that exactly one of them does.
from __future__ import annotations
import asyncio
+import time
from typing import Any
import pytest
@@ -16,6 +17,7 @@ from slack_sdk.errors import SlackApiError
from switch_core.bridges.collaboration.models import InboundCommand
from switch_core.bridges.collaboration.slack.adapter import (
+ _AGENT_SESSIONS_FAILURE_LIMIT,
SlackAdapter,
SlackConnectionConfig,
SlackUser,
@@ -34,7 +36,9 @@ def _run(coro: Any) -> Any:
class FakeResponse(dict):
- pass
+ def __init__(self, *args: Any, headers: dict[str, str] | None = None) -> None:
+ super().__init__(*args)
+ self.headers = headers or {}
class FakeWebClient:
@@ -44,12 +48,16 @@ class FakeWebClient:
self.deleted: list[dict[str, Any]] = []
self.reactions: list[tuple[str, str, str]] = []
self.stream_error: str | None = None
+ self.error_headers: dict[str, str] = {}
self.reaction_error: str | None = None
self._ts = 0
def _fail(self) -> None:
if self.stream_error:
- raise SlackApiError("failed", FakeResponse({"error": self.stream_error}))
+ raise SlackApiError(
+ "failed",
+ FakeResponse({"error": self.stream_error}, headers=self.error_headers),
+ )
async def chat_startStream(self, **kwargs: Any) -> FakeResponse:
self._fail()
@@ -668,3 +676,104 @@ def test_a_root_question_is_marked_on_itself() -> None:
_run(_state(adapter, "working", detail="reading"))
assert ("add", "111.0", "eyes") in client.reactions
+
+
+# ── Throttling and give-ups that expire ──────────────────────────────────────
+
+
+def test_throttling_never_counts_towards_giving_up() -> None:
+ """A busy workspace is not a broken one.
+
+ Counting rate limits as refusals turned a burst of traffic into a bridge
+ that showed no card again until someone restarted it."""
+ adapter, client = _adapter()
+ client.stream_error = "ratelimited"
+
+ for _ in range(_AGENT_SESSIONS_FAILURE_LIMIT + 5):
+ adapter._sessions_throttled_until = None
+ _run(_state(adapter, "working", detail="reading"))
+
+ assert adapter._agent_sessions_off_reason is None
+
+
+def test_throttling_pauses_updates_for_as_long_as_slack_asked() -> None:
+ adapter, client = _adapter()
+ client.stream_error = "ratelimited"
+ client.error_headers = {"Retry-After": "45"}
+
+ _run(_state(adapter, "working", detail="reading"))
+ assert adapter._sessions_throttled_until is not None
+ assert adapter._sessions_throttled_until - time.monotonic() > 40
+
+ client.stream_error = None
+ _run(_state(adapter, "working", detail="still reading"))
+
+ assert "chat.startStream" not in _methods(client)
+
+
+def test_the_card_comes_back_once_the_throttling_window_passes() -> None:
+ adapter, client = _adapter()
+ client.stream_error = "ratelimited"
+ _run(_state(adapter, "working", detail="reading"))
+
+ client.stream_error = None
+ adapter._sessions_throttled_until = time.monotonic() - 1
+ _run(_state(adapter, "working", detail="reading again"))
+
+ assert "chat.startStream" in _methods(client)
+
+
+def test_throttling_is_announced_once_per_burst(
+ caplog: pytest.LogCaptureFixture,
+) -> None:
+ adapter, client = _adapter()
+ client.stream_error = "ratelimited"
+
+ with caplog.at_level("WARNING"):
+ for _ in range(10):
+ _run(_state(adapter, "working", detail="reading"))
+
+ warnings = [r for r in caplog.records if r.levelname == "WARNING"]
+ assert len(warnings) == 1
+ assert "rate-limiting" in warnings[0].getMessage()
+
+
+def test_a_turn_still_ends_while_we_are_being_throttled() -> None:
+ """Skipping teardown would leave the card open for good."""
+ adapter, client = _adapter()
+ _run(_state(adapter, "working", detail="reading"))
+ adapter._sessions_throttled_until = time.monotonic() + 600
+
+ _run(_state(adapter, "idle"))
+
+ assert "chat.stopStream" in _methods(client)
+
+
+def test_a_give_up_over_an_unknown_error_is_retried_later() -> None:
+ adapter, client = _adapter()
+ client.stream_error = "some_code_we_have_never_seen"
+ for _ in range(_AGENT_SESSIONS_FAILURE_LIMIT):
+ _run(_state(adapter, "working", detail="reading"))
+ assert adapter._agent_sessions_off_reason == "some_code_we_have_never_seen"
+
+ client.stream_error = None
+ adapter._agent_sessions_retry_at = time.monotonic() - 1
+ _run(_state(adapter, "working", detail="reading again"))
+
+ assert adapter._agent_sessions_off_reason is None
+ assert "chat.startStream" in _methods(client)
+
+
+def test_a_refusal_that_named_its_cause_is_not_retried() -> None:
+ """Nothing about the workspace changes on its own, so nothing re-arms."""
+ adapter, client = _adapter()
+ client.stream_error = "not_an_agent"
+ _run(_state(adapter, "working", detail="reading"))
+
+ assert adapter._agent_sessions_off_reason == "not_an_agent"
+ assert adapter._agent_sessions_retry_at is None
+
+ client.stream_error = None
+ _run(_state(adapter, "working", detail="reading again"))
+
+ assert "chat.startStream" not in _methods(client)