Open-source AI teammates in Slack, Teams, Discord, Telegram and Mattermost. Self-host it, with any agent harness and any model.
Python agent-orchestrationaiai-agentschatopsdiscordlangchainmcpmicrosoft-teamsmulti-agentopen-sourceopenaiself-hostedslack
Answer a caller that keeps repeating a refused request with 429, from memory (#750)
Abel Dantasa0df95a03ea8
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex ea98eb957..f770b3eee 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -123,6 +123,15 @@ version of their own to them without also giving them a release of their own.rather than per-capability environment variables. Known flags:`agent_management`, `hosted_agents`, `switch_cloud`, and`ecosystem.show_owners`.+- **An agent or controller that keeps repeating a refused request is answered+ 429 from memory.** After 8 identical refusals (401, 403, 404 or 426 for the+ same bearer token, or the same controller credential, on the same method and+ path of an agent or controller route) within two minutes, Switch answers that request 429 with+ `Retry-After` for a minute without looking up its credential or touching+ the database. One request goes through when the wait is over; refused+ again, the wait doubles, up to 15 minutes, and any other answer clears it.+ It protects the server from old clients that retry a refusal forever. New+ metric `switch.http.refusals_throttled`, with a panel on the dashboard.- **Switch cloud machines that run the agents controller sleep when idle,and wake on a message.** Such a machine is stopped once its status reportssay no session has run for `HOSTED_IDLE_STOP_MINUTES` (released, as before,diff --git a/core/switch_core/bridges/agent/app.py b/core/switch_core/bridges/agent/app.pyindex 695aff19a..8e4ebea0f 100644--- a/core/switch_core/bridges/agent/app.py+++ b/core/switch_core/bridges/agent/app.py@@ -1,6 +1,7 @@from __future__ import annotationsimport logging+import timefrom fastapi import FastAPI, HTTPException, Requestfrom fastapi.exceptions import RequestValidationError@@ -38,6 +39,7 @@ from switch_core.db.stores.room_store import RoomStorefrom switch_core.db.stores.task_store import TaskStorefrom switch_core.logging_context import log_contextfrom switch_core.observability.http import MetricsMiddleware+from switch_core.refusal_breaker import RefusalBreakerfrom switch_core.request_context import RequestContextMiddlewarefrom switch_core.room_service import RoomServicefrom switch_core.session_activity.outcomes import ApprovalOutcomes@@ -175,6 +177,9 @@ def create_agent_bridge_app(session_factory=session_factory, # type: ignore[arg-type]controller_auth=controller_auth,)+ # Outside the bearer middleware, so a caller that keeps repeating a refused+ # request is answered before its credential is looked up again.+ app.add_middleware(RefusalBreaker, clock=time.monotonic)# Outside the bearer middleware, so a request rejected for bad credentials# is still counted and timed — an authentication failure is traffic, and a# spike of it is the thing you most want a dashboard to show.diff --git a/core/switch_core/observability/catalogue.py b/core/switch_core/observability/catalogue.pyindex 39d0fa243..b69462372 100644--- a/core/switch_core/observability/catalogue.py+++ b/core/switch_core/observability/catalogue.py@@ -86,6 +86,16 @@ HTTP_REQUEST_DURATION = _spec("route","method",)+# A caller that kept repeating a refused request, answered 429 from memory+# (`refusal_breaker.py`). `route` is the template of the route that refused it.+HTTP_REFUSALS_THROTTLED = _spec(+ "switch.http.refusals_throttled",+ "sum",+ "{request}",+ "Requests answered 429 without being served, because the same caller had "+ "the same request refused repeatedly. By the route that refused it.",+ "route",+)# ── Database ─────────────────────────────────────────────────────────────────DB_POOL_IN_USE = _spec(@@ -590,6 +600,7 @@ CATALOGUE: dict[str, MetricSpec] = {for spec in (HTTP_REQUESTS,HTTP_REQUEST_DURATION,+ HTTP_REFUSALS_THROTTLED,DB_POOL_IN_USE,DB_POOL_SIZE,DB_POOL_OVERFLOW,diff --git a/core/switch_core/refusal_breaker.py b/core/switch_core/refusal_breaker.pynew file mode 100644index 000000000..077c57623--- /dev/null+++ b/core/switch_core/refusal_breaker.py@@ -0,0 +1,206 @@+"""Answers a caller that keeps repeating a refused request, from memory.++Some clients never update and retry a refusal for as long as they run: an+agent runtime asking for a room it is no longer in, a controller on a+protocol this server no longer speaks, a machine whose credential is gone.+Every retry costs a credential lookup at least, and the clients that do it+are the ones that will never be fixed from their side.++After `REFUSALS_TO_TRIP` identical refusals (the same caller, method, path+and status, with nothing but refusals in between) inside `WINDOW_S`, the+breaker opens: that request is answered 429 with `Retry-After` at once,+without reaching authentication, a handler or the database. When the wait is+over one request goes through. If it is refused again the breaker reopens for+twice as long, up to `MAX_COOLDOWN_S`; any answer that is not a refusal+closes it.++The caller is the bearer token, or, for a controller's token exchange, which+carries its credential in the body, the body. Both are hashed, never kept. A+request that names no caller is never throttled, so one client cannot hold+back another. Only agent and controller routes are covered: the gateway+serves people, whose clients can be told what went wrong. The state is this+process's own, which is right while switch-core runs as a single replica.+"""++from __future__ import annotations++import hashlib+import json+import logging+import math+import re+from collections import OrderedDict+from collections.abc import Callable+from dataclasses import dataclass++from starlette.types import ASGIApp, Message, Receive, Scope, Send++from switch_core.observability.catalogue import HTTP_REFUSALS_THROTTLED+from switch_core.observability.http import route_label+from switch_core.observability.metrics import metrics++logger = logging.getLogger(__name__)++REFUSAL_STATUSES = frozenset({401, 403, 404, 426})+REFUSALS_TO_TRIP = 8+WINDOW_S = 120.0+INITIAL_COOLDOWN_S = 60.0+MAX_COOLDOWN_S = 900.0+# Bounds the memory a caller sending a fresh token per request can take.+MAX_TRACKED = 10_000++_COVERED_PREFIXES = ("/agents/", "/agent-sessions/", "/v1/")+_TOKEN_EXCHANGE = re.compile(r"^/v1/management/controllers/[^/]+/token$")+# Controllers read refusals from the management error envelope; everything+# else on this app reads FastAPI's `detail`.+_MANAGEMENT_PREFIX = "/v1/management/"+_RATE_LIMITED = "rate_limited"+++@dataclass+class _Refusals:+ status: int+ count: int+ first_at: float+ route: str+ open_until: float = 0.0+ cooldown_s: float = 0.0+++class RefusalBreaker:+ def __init__(self, app: ASGIApp, *, clock: Callable[[], float]) -> None:+ self.app = app+ self._clock = clock+ self._tracked: OrderedDict[tuple[str, str, str], _Refusals] = OrderedDict()++ async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None:+ path = scope.get("path", "") if scope["type"] == "http" else ""+ if not path.startswith(_COVERED_PREFIXES):+ await self.app(scope, receive, send)+ return+ method = scope.get("method", "GET")+ caller = _bearer(scope)+ if caller is None and method == "POST" and _TOKEN_EXCHANGE.match(path):+ body = await _read_body(receive)+ caller = hashlib.sha256(body).hexdigest()+ receive = _replay(body)+ if caller is None:+ await self.app(scope, receive, send)+ return++ key = (caller, method, path)+ now = self._clock()+ entry = self._tracked.get(key)+ if entry is not None and entry.open_until > now:+ metrics().increment(HTTP_REFUSALS_THROTTLED, {"route": entry.route})+ await _too_many(send, path, math.ceil(entry.open_until - now))+ return++ statuses: list[int] = []++ async def capture(message: Message) -> None:+ if message["type"] == "http.response.start":+ statuses.append(message["status"])+ await send(message)++ await self.app(scope, receive, capture)+ if statuses:+ self._record(key, statuses[0], route_label(scope), method, path)++ def _record(+ self, key: tuple[str, str, str], status: int, route: str, method: str, path: str+ ) -> None:+ if status not in REFUSAL_STATUSES:+ self._tracked.pop(key, None)+ return+ now = self._clock()+ entry = self._tracked.get(key)+ if entry is not None and entry.cooldown_s:+ # The one request let through after a wait was refused again.+ entry.cooldown_s = min(entry.cooldown_s * 2, MAX_COOLDOWN_S)+ entry.open_until = now + entry.cooldown_s+ self._log_open(entry, method, path)+ return+ if entry is None or entry.status != status or now - entry.first_at > WINDOW_S:+ entry = _Refusals(status=status, count=0, first_at=now, route=route)+ self._tracked[key] = entry+ self._tracked.move_to_end(key)+ entry.count += 1+ if entry.count >= REFUSALS_TO_TRIP:+ entry.cooldown_s = INITIAL_COOLDOWN_S+ entry.open_until = now + entry.cooldown_s+ self._log_open(entry, method, path)+ while len(self._tracked) > MAX_TRACKED:+ self._tracked.popitem(last=False)++ def _log_open(self, entry: _Refusals, method: str, path: str) -> None:+ logger.warning(+ "%s %s was refused %d (%d times) by the same caller; answering it 429 for %d s",+ method,+ path,+ entry.status,+ entry.count,+ entry.cooldown_s,+ )+++def _bearer(scope: Scope) -> str | None:+ for name, value in scope.get("headers", []):+ if name == b"authorization" and value.lower().startswith(b"bearer "):+ return hashlib.sha256(value[7:].strip()).hexdigest()+ return None+++async def _read_body(receive: Receive) -> bytes:+ chunks: list[bytes] = []+ while True:+ message = await receive()+ if message["type"] != "http.request":+ return b"".join(chunks)+ chunks.append(message.get("body", b""))+ if not message.get("more_body", False):+ return b"".join(chunks)+++def _replay(body: bytes) -> Receive:+ sent = False++ async def receive() -> Message:+ nonlocal sent+ if sent:+ return {"type": "http.disconnect"}+ sent = True+ return {"type": "http.request", "body": body, "more_body": False}++ return receive+++async def _too_many(send: Send, path: str, retry_after_s: int) -> None:+ message = (+ "This request was refused repeatedly and is not retried for "+ f"{retry_after_s} s. Retrying it unchanged will be refused again."+ )+ if path.startswith(_MANAGEMENT_PREFIX):+ body: dict[str, object] = {+ "error": {+ "code": _RATE_LIMITED,+ "message": message,+ "retryable": True,+ "retry_after_s": retry_after_s,+ }+ }+ else:+ body = {"detail": message}+ payload = json.dumps(body).encode()+ await send(+ {+ "type": "http.response.start",+ "status": 429,+ "headers": [+ (b"content-type", b"application/json"),+ (b"content-length", str(len(payload)).encode()),+ (b"retry-after", str(retry_after_s).encode()),+ ],+ }+ )+ await send({"type": "http.response.body", "body": payload})diff --git a/core/tests/switch_core/test_refusal_breaker.py b/core/tests/switch_core/test_refusal_breaker.pynew file mode 100644index 000000000..879e576c9--- /dev/null+++ b/core/tests/switch_core/test_refusal_breaker.py@@ -0,0 +1,128 @@+import json++from starlette.types import Message, Receive, Scope, Send++from switch_core.refusal_breaker import (+ INITIAL_COOLDOWN_S,+ REFUSALS_TO_TRIP,+ RefusalBreaker,+)+++class _App:+ """Answers every request with `status`, and counts what reached it."""++ def __init__(self, status: int) -> None:+ self.status = status+ self.calls = 0+ self.bodies: list[bytes] = []++ async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None:+ self.calls += 1+ message = await receive()+ self.bodies.append(message.get("body", b""))+ await send(+ {"type": "http.response.start", "status": self.status, "headers": []}+ )+ await send({"type": "http.response.body", "body": b"{}"})+++class _Clock:+ def __init__(self) -> None:+ self.now = 1000.0++ def __call__(self) -> float:+ return self.now+++async def _request(+ breaker: RefusalBreaker,+ path: str,+ *,+ bearer: str | None = "token-a",+ method: str = "GET",+ body: bytes = b"",+) -> tuple[int, dict[bytes, bytes], dict]:+ headers = [(b"authorization", f"Bearer {bearer}".encode())] if bearer else []+ sent: list[Message] = []++ async def receive() -> Message:+ return {"type": "http.request", "body": body, "more_body": False}++ async def send(message: Message) -> None:+ sent.append(message)++ scope = {"type": "http", "method": method, "path": path, "headers": headers}+ await breaker(scope, receive, send)+ start = sent[0]+ payload = json.loads(sent[1]["body"])+ return start["status"], dict(start["headers"]), payload+++async def test_a_repeated_refusal_is_answered_429_without_reaching_the_app() -> None:+ app, clock = _App(403), _Clock()+ breaker = RefusalBreaker(app, clock=clock)+ for _ in range(REFUSALS_TO_TRIP):+ assert (await _request(breaker, "/agents/a1/events"))[0] == 403++ status, headers, payload = await _request(breaker, "/agents/a1/events")+ assert status == 429+ assert headers[b"retry-after"] == str(int(INITIAL_COOLDOWN_S)).encode()+ assert "detail" in payload+ assert app.calls == REFUSALS_TO_TRIP++ # Another caller, and a request naming no caller at all, are not held back.+ assert (await _request(breaker, "/agents/a1/events", bearer="token-b"))[0] == 403+ for _ in range(REFUSALS_TO_TRIP + 2):+ assert (await _request(breaker, "/agents/a1/events", bearer=None))[0] == 403+ assert (await _request(breaker, "/gateway/agents/a1"))[0] == 403+++async def test_after_the_wait_one_request_goes_through_and_its_answer_decides() -> None:+ app, clock = _App(404), _Clock()+ breaker = RefusalBreaker(app, clock=clock)+ for _ in range(REFUSALS_TO_TRIP):+ await _request(breaker, "/agents/a1/connection/beat", method="POST")++ clock.now += INITIAL_COOLDOWN_S+ assert (await _request(breaker, "/agents/a1/connection/beat", method="POST"))[+ 0+ ] == 404+ status, headers, _ = await _request(+ breaker, "/agents/a1/connection/beat", method="POST"+ )+ assert status == 429+ assert headers[b"retry-after"] == str(int(INITIAL_COOLDOWN_S * 2)).encode()++ clock.now += INITIAL_COOLDOWN_S * 2+ app.status = 200+ assert (await _request(breaker, "/agents/a1/connection/beat", method="POST"))[+ 0+ ] == 200+ app.status = 404+ assert (await _request(breaker, "/agents/a1/connection/beat", method="POST"))[+ 0+ ] == 404+++async def test_a_controller_token_exchange_is_told_apart_by_its_credential() -> None:+ app, clock = _App(401), _Clock()+ breaker = RefusalBreaker(app, clock=clock)+ path = "/v1/management/controllers/c1/token"+ stale = b'{"credential": "stale"}'+ for _ in range(REFUSALS_TO_TRIP):+ await _request(breaker, path, bearer=None, method="POST", body=stale)+ assert app.bodies[-1] == stale++ status, _, payload = await _request(+ breaker, path, bearer=None, method="POST", body=stale+ )+ assert status == 429+ assert payload["error"]["code"] == "rate_limited"+ assert payload["error"]["retry_after_s"] == int(INITIAL_COOLDOWN_S)++ app.status = 200+ fresh = b'{"credential": "fresh"}'+ assert (await _request(breaker, path, bearer=None, method="POST", body=fresh))[+ 0+ ] == 200diff --git a/deploy/observability/dashboard.json b/deploy/observability/dashboard.jsonindex 3581f66c0..8cbd9989b 100644--- a/deploy/observability/dashboard.json+++ b/deploy/observability/dashboard.json@@ -824,6 +824,28 @@"height": 4}},+ {+ "definition": {+ "type": "timeseries",+ "title": "Repeated refusals answered 429 per second, by the route that refused them",+ "show_legend": true,+ "requests": [+ {+ "q": "sum:switch.http.refusals_throttled{$env,$service} by {route}.as_rate()",+ "display_type": "bars",+ "style": {+ "palette": "warm"+ }+ }+ ]+ },+ "layout": {+ "x": 0,+ "y": 71,+ "width": 12,+ "height": 4+ }+ },{"definition": {"type": "timeseries",@@ -838,7 +860,7 @@},"layout": {"x": 0,- "y": 71,+ "y": 75,"width": 6,"height": 4}@@ -857,7 +879,7 @@},"layout": {"x": 6,- "y": 71,+ "y": 75,"width": 6,"height": 4}@@ -880,7 +902,7 @@},"layout": {"x": 0,- "y": 75,+ "y": 79,"width": 6,"height": 4}@@ -903,7 +925,7 @@},"layout": {"x": 6,- "y": 75,+ "y": 79,"width": 6,"height": 4}@@ -926,7 +948,7 @@},"layout": {"x": 0,- "y": 79,+ "y": 83,"width": 6,"height": 4}@@ -949,7 +971,7 @@},"layout": {"x": 6,- "y": 79,+ "y": 83,"width": 6,"height": 4}@@ -968,7 +990,7 @@},"layout": {"x": 0,- "y": 83,+ "y": 87,"width": 6,"height": 4}@@ -987,7 +1009,7 @@},"layout": {"x": 6,- "y": 83,+ "y": 87,"width": 6,"height": 4}@@ -1007,7 +1029,7 @@},"layout": {"x": 0,- "y": 87,+ "y": 91,"width": 12,"height": 2}@@ -1026,7 +1048,7 @@},"layout": {"x": 0,- "y": 89,+ "y": 93,"width": 6,"height": 4}@@ -1053,7 +1075,7 @@},"layout": {"x": 6,- "y": 89,+ "y": 93,"width": 6,"height": 4}@@ -1076,7 +1098,7 @@},"layout": {"x": 0,- "y": 93,+ "y": 97,"width": 6,"height": 4}@@ -1095,7 +1117,7 @@},"layout": {"x": 6,- "y": 93,+ "y": 97,"width": 6,"height": 4}