Skip to content

Open-source AI teammates in Slack, Teams, Discord, Telegram and Mattermost. Self-host it, with any agent harness and any model.

Answer a caller that keeps repeating a refused request with 429, from memory (#750)

Abel Dantasa0df95a03ea8
diff --git a/CHANGELOG.md b/CHANGELOG.md
index ea98eb957..f770b3eee 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -123,6 +123,15 @@ version of their own to them without also giving them a release of their own.
rather than per-capability environment variables. Known flags:
`agent_management`, `hosted_agents`, `switch_cloud`, and
`ecosystem.show_owners`.
+- **An agent or controller that keeps repeating a refused request is answered
+ 429 from memory.** After 8 identical refusals (401, 403, 404 or 426 for the
+ same bearer token, or the same controller credential, on the same method and
+ path of an agent or controller route) within two minutes, Switch answers that request 429 with
+ `Retry-After` for a minute without looking up its credential or touching
+ the database. One request goes through when the wait is over; refused
+ again, the wait doubles, up to 15 minutes, and any other answer clears it.
+ It protects the server from old clients that retry a refusal forever. New
+ metric `switch.http.refusals_throttled`, with a panel on the dashboard.
- **Switch cloud machines that run the agents controller sleep when idle,
and wake on a message.** Such a machine is stopped once its status reports
say no session has run for `HOSTED_IDLE_STOP_MINUTES` (released, as before,
diff --git a/core/switch_core/bridges/agent/app.py b/core/switch_core/bridges/agent/app.py
index 695aff19a..8e4ebea0f 100644
--- a/core/switch_core/bridges/agent/app.py
+++ b/core/switch_core/bridges/agent/app.py
@@ -1,6 +1,7 @@
from __future__ import annotations
import logging
+import time
from fastapi import FastAPI, HTTPException, Request
from fastapi.exceptions import RequestValidationError
@@ -38,6 +39,7 @@ from switch_core.db.stores.room_store import RoomStore
from switch_core.db.stores.task_store import TaskStore
from switch_core.logging_context import log_context
from switch_core.observability.http import MetricsMiddleware
+from switch_core.refusal_breaker import RefusalBreaker
from switch_core.request_context import RequestContextMiddleware
from switch_core.room_service import RoomService
from switch_core.session_activity.outcomes import ApprovalOutcomes
@@ -175,6 +177,9 @@ def create_agent_bridge_app(
session_factory=session_factory, # type: ignore[arg-type]
controller_auth=controller_auth,
)
+ # Outside the bearer middleware, so a caller that keeps repeating a refused
+ # request is answered before its credential is looked up again.
+ app.add_middleware(RefusalBreaker, clock=time.monotonic)
# Outside the bearer middleware, so a request rejected for bad credentials
# is still counted and timed — an authentication failure is traffic, and a
# spike of it is the thing you most want a dashboard to show.
diff --git a/core/switch_core/observability/catalogue.py b/core/switch_core/observability/catalogue.py
index 39d0fa243..b69462372 100644
--- a/core/switch_core/observability/catalogue.py
+++ b/core/switch_core/observability/catalogue.py
@@ -86,6 +86,16 @@ HTTP_REQUEST_DURATION = _spec(
"route",
"method",
)
+# A caller that kept repeating a refused request, answered 429 from memory
+# (`refusal_breaker.py`). `route` is the template of the route that refused it.
+HTTP_REFUSALS_THROTTLED = _spec(
+ "switch.http.refusals_throttled",
+ "sum",
+ "{request}",
+ "Requests answered 429 without being served, because the same caller had "
+ "the same request refused repeatedly. By the route that refused it.",
+ "route",
+)
# ── Database ─────────────────────────────────────────────────────────────────
DB_POOL_IN_USE = _spec(
@@ -590,6 +600,7 @@ CATALOGUE: dict[str, MetricSpec] = {
for spec in (
HTTP_REQUESTS,
HTTP_REQUEST_DURATION,
+ HTTP_REFUSALS_THROTTLED,
DB_POOL_IN_USE,
DB_POOL_SIZE,
DB_POOL_OVERFLOW,
diff --git a/core/switch_core/refusal_breaker.py b/core/switch_core/refusal_breaker.py
new file mode 100644
index 000000000..077c57623
--- /dev/null
+++ b/core/switch_core/refusal_breaker.py
@@ -0,0 +1,206 @@
+"""Answers a caller that keeps repeating a refused request, from memory.
+
+Some clients never update and retry a refusal for as long as they run: an
+agent runtime asking for a room it is no longer in, a controller on a
+protocol this server no longer speaks, a machine whose credential is gone.
+Every retry costs a credential lookup at least, and the clients that do it
+are the ones that will never be fixed from their side.
+
+After `REFUSALS_TO_TRIP` identical refusals (the same caller, method, path
+and status, with nothing but refusals in between) inside `WINDOW_S`, the
+breaker opens: that request is answered 429 with `Retry-After` at once,
+without reaching authentication, a handler or the database. When the wait is
+over one request goes through. If it is refused again the breaker reopens for
+twice as long, up to `MAX_COOLDOWN_S`; any answer that is not a refusal
+closes it.
+
+The caller is the bearer token, or, for a controller's token exchange, which
+carries its credential in the body, the body. Both are hashed, never kept. A
+request that names no caller is never throttled, so one client cannot hold
+back another. Only agent and controller routes are covered: the gateway
+serves people, whose clients can be told what went wrong. The state is this
+process's own, which is right while switch-core runs as a single replica.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import logging
+import math
+import re
+from collections import OrderedDict
+from collections.abc import Callable
+from dataclasses import dataclass
+
+from starlette.types import ASGIApp, Message, Receive, Scope, Send
+
+from switch_core.observability.catalogue import HTTP_REFUSALS_THROTTLED
+from switch_core.observability.http import route_label
+from switch_core.observability.metrics import metrics
+
+logger = logging.getLogger(__name__)
+
+REFUSAL_STATUSES = frozenset({401, 403, 404, 426})
+REFUSALS_TO_TRIP = 8
+WINDOW_S = 120.0
+INITIAL_COOLDOWN_S = 60.0
+MAX_COOLDOWN_S = 900.0
+# Bounds the memory a caller sending a fresh token per request can take.
+MAX_TRACKED = 10_000
+
+_COVERED_PREFIXES = ("/agents/", "/agent-sessions/", "/v1/")
+_TOKEN_EXCHANGE = re.compile(r"^/v1/management/controllers/[^/]+/token$")
+# Controllers read refusals from the management error envelope; everything
+# else on this app reads FastAPI's `detail`.
+_MANAGEMENT_PREFIX = "/v1/management/"
+_RATE_LIMITED = "rate_limited"
+
+
+@dataclass
+class _Refusals:
+ status: int
+ count: int
+ first_at: float
+ route: str
+ open_until: float = 0.0
+ cooldown_s: float = 0.0
+
+
+class RefusalBreaker:
+ def __init__(self, app: ASGIApp, *, clock: Callable[[], float]) -> None:
+ self.app = app
+ self._clock = clock
+ self._tracked: OrderedDict[tuple[str, str, str], _Refusals] = OrderedDict()
+
+ async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None:
+ path = scope.get("path", "") if scope["type"] == "http" else ""
+ if not path.startswith(_COVERED_PREFIXES):
+ await self.app(scope, receive, send)
+ return
+ method = scope.get("method", "GET")
+ caller = _bearer(scope)
+ if caller is None and method == "POST" and _TOKEN_EXCHANGE.match(path):
+ body = await _read_body(receive)
+ caller = hashlib.sha256(body).hexdigest()
+ receive = _replay(body)
+ if caller is None:
+ await self.app(scope, receive, send)
+ return
+
+ key = (caller, method, path)
+ now = self._clock()
+ entry = self._tracked.get(key)
+ if entry is not None and entry.open_until > now:
+ metrics().increment(HTTP_REFUSALS_THROTTLED, {"route": entry.route})
+ await _too_many(send, path, math.ceil(entry.open_until - now))
+ return
+
+ statuses: list[int] = []
+
+ async def capture(message: Message) -> None:
+ if message["type"] == "http.response.start":
+ statuses.append(message["status"])
+ await send(message)
+
+ await self.app(scope, receive, capture)
+ if statuses:
+ self._record(key, statuses[0], route_label(scope), method, path)
+
+ def _record(
+ self, key: tuple[str, str, str], status: int, route: str, method: str, path: str
+ ) -> None:
+ if status not in REFUSAL_STATUSES:
+ self._tracked.pop(key, None)
+ return
+ now = self._clock()
+ entry = self._tracked.get(key)
+ if entry is not None and entry.cooldown_s:
+ # The one request let through after a wait was refused again.
+ entry.cooldown_s = min(entry.cooldown_s * 2, MAX_COOLDOWN_S)
+ entry.open_until = now + entry.cooldown_s
+ self._log_open(entry, method, path)
+ return
+ if entry is None or entry.status != status or now - entry.first_at > WINDOW_S:
+ entry = _Refusals(status=status, count=0, first_at=now, route=route)
+ self._tracked[key] = entry
+ self._tracked.move_to_end(key)
+ entry.count += 1
+ if entry.count >= REFUSALS_TO_TRIP:
+ entry.cooldown_s = INITIAL_COOLDOWN_S
+ entry.open_until = now + entry.cooldown_s
+ self._log_open(entry, method, path)
+ while len(self._tracked) > MAX_TRACKED:
+ self._tracked.popitem(last=False)
+
+ def _log_open(self, entry: _Refusals, method: str, path: str) -> None:
+ logger.warning(
+ "%s %s was refused %d (%d times) by the same caller; answering it 429 for %d s",
+ method,
+ path,
+ entry.status,
+ entry.count,
+ entry.cooldown_s,
+ )
+
+
+def _bearer(scope: Scope) -> str | None:
+ for name, value in scope.get("headers", []):
+ if name == b"authorization" and value.lower().startswith(b"bearer "):
+ return hashlib.sha256(value[7:].strip()).hexdigest()
+ return None
+
+
+async def _read_body(receive: Receive) -> bytes:
+ chunks: list[bytes] = []
+ while True:
+ message = await receive()
+ if message["type"] != "http.request":
+ return b"".join(chunks)
+ chunks.append(message.get("body", b""))
+ if not message.get("more_body", False):
+ return b"".join(chunks)
+
+
+def _replay(body: bytes) -> Receive:
+ sent = False
+
+ async def receive() -> Message:
+ nonlocal sent
+ if sent:
+ return {"type": "http.disconnect"}
+ sent = True
+ return {"type": "http.request", "body": body, "more_body": False}
+
+ return receive
+
+
+async def _too_many(send: Send, path: str, retry_after_s: int) -> None:
+ message = (
+ "This request was refused repeatedly and is not retried for "
+ f"{retry_after_s} s. Retrying it unchanged will be refused again."
+ )
+ if path.startswith(_MANAGEMENT_PREFIX):
+ body: dict[str, object] = {
+ "error": {
+ "code": _RATE_LIMITED,
+ "message": message,
+ "retryable": True,
+ "retry_after_s": retry_after_s,
+ }
+ }
+ else:
+ body = {"detail": message}
+ payload = json.dumps(body).encode()
+ await send(
+ {
+ "type": "http.response.start",
+ "status": 429,
+ "headers": [
+ (b"content-type", b"application/json"),
+ (b"content-length", str(len(payload)).encode()),
+ (b"retry-after", str(retry_after_s).encode()),
+ ],
+ }
+ )
+ await send({"type": "http.response.body", "body": payload})
diff --git a/core/tests/switch_core/test_refusal_breaker.py b/core/tests/switch_core/test_refusal_breaker.py
new file mode 100644
index 000000000..879e576c9
--- /dev/null
+++ b/core/tests/switch_core/test_refusal_breaker.py
@@ -0,0 +1,128 @@
+import json
+
+from starlette.types import Message, Receive, Scope, Send
+
+from switch_core.refusal_breaker import (
+ INITIAL_COOLDOWN_S,
+ REFUSALS_TO_TRIP,
+ RefusalBreaker,
+)
+
+
+class _App:
+ """Answers every request with `status`, and counts what reached it."""
+
+ def __init__(self, status: int) -> None:
+ self.status = status
+ self.calls = 0
+ self.bodies: list[bytes] = []
+
+ async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None:
+ self.calls += 1
+ message = await receive()
+ self.bodies.append(message.get("body", b""))
+ await send(
+ {"type": "http.response.start", "status": self.status, "headers": []}
+ )
+ await send({"type": "http.response.body", "body": b"{}"})
+
+
+class _Clock:
+ def __init__(self) -> None:
+ self.now = 1000.0
+
+ def __call__(self) -> float:
+ return self.now
+
+
+async def _request(
+ breaker: RefusalBreaker,
+ path: str,
+ *,
+ bearer: str | None = "token-a",
+ method: str = "GET",
+ body: bytes = b"",
+) -> tuple[int, dict[bytes, bytes], dict]:
+ headers = [(b"authorization", f"Bearer {bearer}".encode())] if bearer else []
+ sent: list[Message] = []
+
+ async def receive() -> Message:
+ return {"type": "http.request", "body": body, "more_body": False}
+
+ async def send(message: Message) -> None:
+ sent.append(message)
+
+ scope = {"type": "http", "method": method, "path": path, "headers": headers}
+ await breaker(scope, receive, send)
+ start = sent[0]
+ payload = json.loads(sent[1]["body"])
+ return start["status"], dict(start["headers"]), payload
+
+
+async def test_a_repeated_refusal_is_answered_429_without_reaching_the_app() -> None:
+ app, clock = _App(403), _Clock()
+ breaker = RefusalBreaker(app, clock=clock)
+ for _ in range(REFUSALS_TO_TRIP):
+ assert (await _request(breaker, "/agents/a1/events"))[0] == 403
+
+ status, headers, payload = await _request(breaker, "/agents/a1/events")
+ assert status == 429
+ assert headers[b"retry-after"] == str(int(INITIAL_COOLDOWN_S)).encode()
+ assert "detail" in payload
+ assert app.calls == REFUSALS_TO_TRIP
+
+ # Another caller, and a request naming no caller at all, are not held back.
+ assert (await _request(breaker, "/agents/a1/events", bearer="token-b"))[0] == 403
+ for _ in range(REFUSALS_TO_TRIP + 2):
+ assert (await _request(breaker, "/agents/a1/events", bearer=None))[0] == 403
+ assert (await _request(breaker, "/gateway/agents/a1"))[0] == 403
+
+
+async def test_after_the_wait_one_request_goes_through_and_its_answer_decides() -> None:
+ app, clock = _App(404), _Clock()
+ breaker = RefusalBreaker(app, clock=clock)
+ for _ in range(REFUSALS_TO_TRIP):
+ await _request(breaker, "/agents/a1/connection/beat", method="POST")
+
+ clock.now += INITIAL_COOLDOWN_S
+ assert (await _request(breaker, "/agents/a1/connection/beat", method="POST"))[
+ 0
+ ] == 404
+ status, headers, _ = await _request(
+ breaker, "/agents/a1/connection/beat", method="POST"
+ )
+ assert status == 429
+ assert headers[b"retry-after"] == str(int(INITIAL_COOLDOWN_S * 2)).encode()
+
+ clock.now += INITIAL_COOLDOWN_S * 2
+ app.status = 200
+ assert (await _request(breaker, "/agents/a1/connection/beat", method="POST"))[
+ 0
+ ] == 200
+ app.status = 404
+ assert (await _request(breaker, "/agents/a1/connection/beat", method="POST"))[
+ 0
+ ] == 404
+
+
+async def test_a_controller_token_exchange_is_told_apart_by_its_credential() -> None:
+ app, clock = _App(401), _Clock()
+ breaker = RefusalBreaker(app, clock=clock)
+ path = "/v1/management/controllers/c1/token"
+ stale = b'{"credential": "stale"}'
+ for _ in range(REFUSALS_TO_TRIP):
+ await _request(breaker, path, bearer=None, method="POST", body=stale)
+ assert app.bodies[-1] == stale
+
+ status, _, payload = await _request(
+ breaker, path, bearer=None, method="POST", body=stale
+ )
+ assert status == 429
+ assert payload["error"]["code"] == "rate_limited"
+ assert payload["error"]["retry_after_s"] == int(INITIAL_COOLDOWN_S)
+
+ app.status = 200
+ fresh = b'{"credential": "fresh"}'
+ assert (await _request(breaker, path, bearer=None, method="POST", body=fresh))[
+ 0
+ ] == 200
diff --git a/deploy/observability/dashboard.json b/deploy/observability/dashboard.json
index 3581f66c0..8cbd9989b 100644
--- a/deploy/observability/dashboard.json
+++ b/deploy/observability/dashboard.json
@@ -824,6 +824,28 @@
"height": 4
}
},
+ {
+ "definition": {
+ "type": "timeseries",
+ "title": "Repeated refusals answered 429 per second, by the route that refused them",
+ "show_legend": true,
+ "requests": [
+ {
+ "q": "sum:switch.http.refusals_throttled{$env,$service} by {route}.as_rate()",
+ "display_type": "bars",
+ "style": {
+ "palette": "warm"
+ }
+ }
+ ]
+ },
+ "layout": {
+ "x": 0,
+ "y": 71,
+ "width": 12,
+ "height": 4
+ }
+ },
{
"definition": {
"type": "timeseries",
@@ -838,7 +860,7 @@
},
"layout": {
"x": 0,
- "y": 71,
+ "y": 75,
"width": 6,
"height": 4
}
@@ -857,7 +879,7 @@
},
"layout": {
"x": 6,
- "y": 71,
+ "y": 75,
"width": 6,
"height": 4
}
@@ -880,7 +902,7 @@
},
"layout": {
"x": 0,
- "y": 75,
+ "y": 79,
"width": 6,
"height": 4
}
@@ -903,7 +925,7 @@
},
"layout": {
"x": 6,
- "y": 75,
+ "y": 79,
"width": 6,
"height": 4
}
@@ -926,7 +948,7 @@
},
"layout": {
"x": 0,
- "y": 79,
+ "y": 83,
"width": 6,
"height": 4
}
@@ -949,7 +971,7 @@
},
"layout": {
"x": 6,
- "y": 79,
+ "y": 83,
"width": 6,
"height": 4
}
@@ -968,7 +990,7 @@
},
"layout": {
"x": 0,
- "y": 83,
+ "y": 87,
"width": 6,
"height": 4
}
@@ -987,7 +1009,7 @@
},
"layout": {
"x": 6,
- "y": 83,
+ "y": 87,
"width": 6,
"height": 4
}
@@ -1007,7 +1029,7 @@
},
"layout": {
"x": 0,
- "y": 87,
+ "y": 91,
"width": 12,
"height": 2
}
@@ -1026,7 +1048,7 @@
},
"layout": {
"x": 0,
- "y": 89,
+ "y": 93,
"width": 6,
"height": 4
}
@@ -1053,7 +1075,7 @@
},
"layout": {
"x": 6,
- "y": 89,
+ "y": 93,
"width": 6,
"height": 4
}
@@ -1076,7 +1098,7 @@
},
"layout": {
"x": 0,
- "y": 93,
+ "y": 97,
"width": 6,
"height": 4
}
@@ -1095,7 +1117,7 @@
},
"layout": {
"x": 6,
- "y": 93,
+ "y": 97,
"width": 6,
"height": 4
}