Open-source AI teammates in Slack, Teams, Discord, Telegram and Mattermost. Self-host it, with any agent harness and any model.
Python agent-orchestrationaiai-agentschatopsdiscordlangchainmcpmicrosoft-teamsmulti-agentopen-sourceopenaiself-hostedslack
"""Paces connection opens, so a reconnect storm becomes a queue.
When a pod goes away, every agent and controller it held opens a connection on
the pods that are left, all within a few seconds. Each open authenticates,
makes a new incarnation of the connection and reads where the stream starts:
dozens of transactions, on the same pool the process's requests and its own
lease renewal use. Measured on staging (two pods, 1,000 agents, one pod
deleted), the opens took the whole pool: thousands of pool timeouts, opens
that failed after their incarnation was made, and the surviving pod's lease
lapsing, which made it drop agents of its own.
This middleware runs at most `limit` opens at once in this process. An open
that finds every slot taken waits up to `WAIT_S` for one. Past that, or when
more opens are waiting already than can get a slot within `WAIT_S`, it is
refused before authentication or the database, with `retry_after_s`: its place
in the line. The line moves one slot every median open time over `limit` (the
median of the last `limit` opens), so a storm comes back spread over the time
the process needs to take it, at the pace opens are going here, and the line
slows down when the database does. A caller refused again before its time
keeps that time, so a client that retries at once cannot push everyone else's
back.
An open is done when its first frame or response body goes out (the
`connection_state`, or a refusal) or the request ends. What follows is the
stream, which the event buffer's own cap paces. The state is this process's
own: each pod paces its own opens, as each has its own pool.
"""
from __future__ import annotations
import asyncio
import contextlib
import json
import logging
import math
import random
import re
import statistics
from collections import OrderedDict, deque
from collections.abc import Callable
from dataclasses import dataclass
from starlette.types import ASGIApp, Message, Receive, Scope, Send
from switch_core.observability.catalogue import AGENT_OPENS_DEFERRED
from switch_core.observability.metrics import metrics
from switch_core.refusal_breaker import bearer_caller
logger = logging.getLogger(__name__)
# How long an open waits for a slot before it is told when to come back. The
# wait and the open after it have to fit well inside the timeout of the
# proxies in front of switch-core: the tailnet ingress gives up on a request
# after about 10 s (measured on dev-abel, 29 Oct 2026). An open that outwaits
# a proxy still runs here after the proxy has given up, which moves the
# connection on behind its client's back, and today's agent runtime gives up
# for good on the refusal that follows. 2 s leaves most of the 10 s to the open
# itself and to proxies that give up sooner.
WAIT_S = 2.0
# The callers whose come-back time is remembered, which bounds this memory to a
# few MB. The gate answers before authentication, so a caller that never
# authenticates takes a place in the line too; the line given out goes no
# further than these callers' worth of slots, which bounds how far such
# callers can push back the times real ones are given. 10,000 is about the
# most agents one deployment has been tested with.
MAX_TRACKED = 10_000
BUSY = "busy"
_STATUS = 503
# The opens it paces, by the label its metric carries.
_ROUTES: tuple[tuple[re.Pattern[str], str], ...] = (
(re.compile(r"^/agents/[^/]+/connection/ws$"), "agent_socket"),
(re.compile(r"^/agents/[^/]+/events$"), "agent_event_stream"),
(re.compile(r"^/v1/controllers/[^/]+/connection$"), "controller_open"),
(re.compile(r"^/v1/controllers/[^/]+/connection/ws$"), "controller_socket"),
)
# Controllers read refusals from the management error envelope; everything
# else on this app reads FastAPI's `detail`.
_MANAGEMENT_PREFIXES = ("/v1/controllers/", "/v1/management/")
def _route(scope: Scope) -> str | None:
if scope["type"] not in ("http", "websocket"):
return None
path = scope.get("path", "")
if scope["type"] == "http" and scope.get("method") not in ("GET", "POST"):
return None
for pattern, label in _ROUTES:
if pattern.match(path):
return label
return None
@dataclass(eq=False)
class _Waiter:
caller: str | None
# Set when a slot is handed to this open.
slot: asyncio.Future[None]
class OpenGate:
def __init__(
self,
app: ASGIApp,
*,
limit: int,
clock: Callable[[], float],
rand: Callable[[], float] = random.random,
) -> None:
self.app = app
self.limit = limit
self._clock = clock
self._rand = rand
self._slots = max(limit, 1)
self._running = 0
self._waiting: deque[_Waiter] = deque()
# How long the last `limit` opens took, from taking a slot to their
# first frame or response.
self._open_s: deque[float] = deque(maxlen=self._slots)
# The end of the line of callers told when to come back.
self._next_at = 0.0
self._given: OrderedDict[str, float] = OrderedDict()
self._refusing_since: float | None = None
self._refused = 0
async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None:
route = _route(scope) if self.limit > 0 else None
if route is None:
await self.app(scope, receive, send)
return
retry_after_s = await self._enter(bearer_caller(scope))
if retry_after_s is not None:
metrics().increment(AGENT_OPENS_DEFERRED, {"route": route})
self._note_refused(retry_after_s)
await _refuse(scope, receive, send, retry_after_s)
return
started = self._clock()
released = False
def release() -> None:
nonlocal released
if not released:
released = True
self._release(max(self._clock() - started, 0.0))
async def done_on_first_output(message: Message) -> None:
if message["type"] in (
"websocket.send",
"websocket.close",
"http.response.body",
):
release()
await send(message)
try:
await self.app(scope, receive, done_on_first_output)
finally:
release()
async def _enter(self, caller: str | None) -> float | None:
"""Take a slot, waiting up to `WAIT_S` for one. None once the open holds
a slot; otherwise when it is to come back, in seconds."""
if self._running < self._slots and not self._waiting:
self._running += 1
self._forget(caller)
return None
if len(self._waiting) >= self._most_waiting():
return self._retry_after(caller, self._clock())
waiter = _Waiter(caller, asyncio.get_running_loop().create_future())
self._waiting.append(waiter)
try:
async with asyncio.timeout(WAIT_S):
await waiter.slot
except TimeoutError:
if not waiter.slot.done() or waiter.slot.cancelled():
with contextlib.suppress(ValueError):
self._waiting.remove(waiter)
return self._retry_after(caller, self._clock())
# Handed a slot as the wait ran out: it is this open's.
except asyncio.CancelledError:
if waiter.slot.cancelled():
with contextlib.suppress(ValueError):
self._waiting.remove(waiter)
else:
# Handed a slot as the request went away: the next one has it.
self._pass_slot()
raise
self._forget(caller)
return None
def _release(self, took: float) -> None:
self._open_s.append(took)
self._pass_slot()
self._note_line_through()
def _pass_slot(self) -> None:
while self._waiting:
waiter = self._waiting.popleft()
if not waiter.slot.done():
waiter.slot.set_result(None)
return
self._running -= 1
def _open_time(self) -> float:
"""How long an open takes here: the median of the last `limit`. Before
any has finished, at least `WAIT_S`, the time an open has just waited
with every slot taken and none of them done."""
return statistics.median(self._open_s) if self._open_s else WAIT_S
def _most_waiting(self) -> float:
"""The most opens worth keeping waiting: as many as get a slot within
`WAIT_S` at the pace opens are going here. An open past them would
wait out `WAIT_S` and be refused anyway, so it is refused at once.
Before any open has finished there is no pace to go by, and `WAIT_S`
alone bounds the wait."""
if not self._open_s or self._open_time() <= 0:
return math.inf
return WAIT_S * self._slots / self._open_time()
def _retry_after(self, caller: str | None, now: float) -> float:
"""When this caller is to come back, in seconds: its place in the line.
The line moves one slot every open time over `limit`. The first caller
told to come back is behind the opens waiting here, each one after it a
slot later, at a random point within its slot. The line reaches no
further than `MAX_TRACKED` callers' worth of slots; past that, a caller
is given a random point within it. A caller already given a time that
has not come keeps it."""
if caller is not None:
at = self._given.get(caller)
if at is not None and at > now:
self._given.move_to_end(caller)
return at - now
spacing = self._open_time() / self._slots
end = now + MAX_TRACKED * spacing
start = max(now + (len(self._waiting) + 1) * spacing, self._next_at)
if start < end:
self._next_at = start + spacing
at = start + spacing * self._rand()
else:
# Past the end of the line: a random point within it, so callers
# refused together do not come back together.
self._next_at = end
at = now + spacing + (end - now - spacing) * self._rand()
if caller is not None:
self._given[caller] = at
self._given.move_to_end(caller)
while len(self._given) > MAX_TRACKED:
self._given.popitem(last=False)
return at - now
def _forget(self, caller: str | None) -> None:
if caller is not None:
self._given.pop(caller, None)
def _note_refused(self, retry_after_s: float) -> None:
self._refused += 1
if self._refusing_since is not None:
return
self._refusing_since = self._clock()
logger.warning(
"Opening connections more slowly than they arrive: %d opening and %d "
"waiting for a slot, the rest told when to come back, the first in %.1f s",
self._running,
len(self._waiting),
retry_after_s,
)
def _note_line_through(self) -> None:
now = self._clock()
if self._refusing_since is None or self._waiting or self._next_at > now:
return
logger.warning(
"Caught up with connection opens: %d were told to come back later "
"over %.0f s",
self._refused,
now - self._refusing_since,
)
self._refusing_since = None
self._refused = 0
async def _refuse(
scope: Scope, receive: Receive, send: Send, retry_after_s: float
) -> None:
"""Refuse the open in the form its client reads refusals in, with when to
come back."""
# Rounded up to the millisecond clients wait in, so none comes back early.
seconds = math.ceil(retry_after_s * 1000) / 1000
message = (
"Switch is opening as many connections as it can take right now. "
f"Open again in {seconds:g} s."
)
path = scope.get("path", "")
if scope["type"] == "websocket":
# As every refused agent or controller socket: accepted, a `refused`
# frame with the status and detail, then closed with 4000 plus it.
await receive() # the client's websocket.connect
await send({"type": "websocket.accept"})
frame = {
"event": "refused",
"data": {
"status": _STATUS,
"detail": {
"code": BUSY,
"message": message,
"retry_after_s": seconds,
},
},
}
await send({"type": "websocket.send", "text": json.dumps(frame)})
await send({"type": "websocket.close", "code": 4000 + _STATUS})
return
if path.startswith(_MANAGEMENT_PREFIXES):
body: dict[str, object] = {
"error": {
"code": BUSY,
"message": message,
"retryable": True,
"retry_after_s": seconds,
}
}
else:
body = {"detail": {"code": BUSY, "message": message, "retry_after_s": seconds}}
payload = json.dumps(body).encode()
await send(
{
"type": "http.response.start",
"status": _STATUS,
"headers": [
(b"content-type", b"application/json"),
(b"retry-after", str(math.ceil(retry_after_s)).encode()),
(b"content-length", str(len(payload)).encode()),
],
}
)
await send({"type": "http.response.body", "body": payload})