Open-source AI teammates in Slack, Teams, Discord, Telegram and Mattermost. Self-host it, with any agent harness and any model.
Python agent-orchestrationaiai-agentschatopsdiscordlangchainmcpmicrosoft-teamsmulti-agentopen-sourceopenaiself-hostedslack
"""Server-side observation points for the end-to-end benchmark.
Everything here wraps objects the harness itself owns — the ASGI app returned
by the real application factory, and the engine the harness built. No module
under `switch_core` is modified, imported differently, or monkeypatched, so a
benchmark run exercises the same server code a deployment runs.
"""
from __future__ import annotations
import json
import re
from collections import Counter
from collections.abc import Awaitable, Callable
from contextvars import ContextVar
from typing import Any
from sqlalchemy import event
from sqlalchemy.ext.asyncio import AsyncEngine
from tests.benchmarks.trace import (
APPROVAL_OPENED,
REPLY_ACCEPTED,
REPLY_COMMITTED,
REPLY_RECEIVED,
SSE_PUSH,
TURN_REPORTED,
TraceCollector,
correlation_for,
)
Scope = dict[str, Any]
Receive = Callable[[], Awaitable[dict[str, Any]]]
Send = Callable[[dict[str, Any]], Awaitable[None]]
_EVENT_STREAM = re.compile(r"^/agents/(?P<agent>[^/]+)/events$")
_POST_MESSAGE = re.compile(r"^/agents/(?P<agent>[^/]+)/ops/post_message$")
_ACTIVITY = re.compile(r"^/agent-sessions/(?P<session>[^/]+)/activity$")
_APPROVALS = re.compile(r"^/agent-sessions/(?P<session>[^/]+)/approvals$")
#: The reply the benchmark provider posts, as `bench/adapter.ts` writes it. The
#: room and message are the ones its prompt named, so the reply can be matched
#: to the message it answers; the room it lands in is read back separately.
REPLY = re.compile(
r"switch-bench-reply:(?P<marker>\S+) room=(?P<room>\S+) message=(?P<message>\S+)"
)
#: Statuses a turn row ends on.
_TURN_ENDED = frozenset({"completed", "interrupted", "error"})
# The reply request in flight on this task, so an engine-level commit can be
# attributed to the message it answers. Set by the ASGI wrapper and read by the
# commit listener; unset for every other request, which is how commits from
# unrelated work stay out of the measurement.
_in_flight: ContextVar[str | None] = ContextVar(
"benchmark_in_flight_correlation", default=None
)
def reply_correlation(text: str) -> str | None:
"""The message a benchmark reply answers, or None for any other text."""
matched = REPLY.search(text)
if matched is None:
return None
return correlation_for(matched["room"], matched["message"])
def _sse_correlations(chunk: bytes) -> list[str]:
"""Correlations for every room message in one SSE write.
A single ASGI body chunk may carry several frames, and most frames are not
room messages at all — `connection_state`, `gap`, `evicted`,
`room_released`, keepalive comments. The measured workload is addressed
room messages, so only `message` frames count; the others have no message
id to correlate on and are not what the latency figures describe.
`room_id` sits on the event, `message_id` inside its payload — the shape
`AgentEvent` and `MessagePayload` define.
"""
found: list[str] = []
for block in chunk.split(b"\n\n"):
if not block.strip() or block.lstrip().startswith(b":"):
continue
name = None
data: dict[str, Any] | None = None
for line in block.split(b"\n"):
if line.startswith(b"event: "):
name = line[len(b"event: ") :].decode()
elif line.startswith(b"data: "):
try:
parsed = json.loads(line[len(b"data: ") :])
except ValueError:
continue
if isinstance(parsed, dict):
data = parsed
if name != "message" or data is None:
continue
room_id = data.get("room_id")
payload = data.get("payload")
message_id = payload.get("message_id") if isinstance(payload, dict) else None
if isinstance(room_id, str) and isinstance(message_id, str):
found.append(correlation_for(room_id, message_id))
return found
class TracingMiddleware:
"""Pure-ASGI wrapper recording the server-side HTTP points.
Deliberately ASGI rather than a Starlette `BaseHTTPMiddleware`: the latter
buffers a streaming response through a queue, which would both change the
timing being measured and defeat the point of an SSE stream.
"""
def __init__(self, app: Any, collector: TraceCollector) -> None:
self._app = app
self._collector = collector
async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None:
if scope.get("type") != "http":
await self._app(scope, receive, send)
return
path = scope.get("path", "")
method = scope.get("method")
if method == "GET" and _EVENT_STREAM.match(path):
await self._stream(scope, receive, send)
elif method == "POST" and _POST_MESSAGE.match(path):
await self._reply(scope, receive, send)
elif method == "POST" and _ACTIVITY.match(path):
await self._activity(scope, receive, send)
elif method == "POST" and _APPROVALS.match(path):
await self._approval(scope, receive, send)
else:
await self._app(scope, receive, send)
async def _reply(self, scope: Scope, receive: Receive, send: Send) -> None:
body, replay = await _buffer_body(receive)
payload = _json_object(body)
text = payload.get("body") if payload is not None else None
correlation = reply_correlation(text) if isinstance(text, str) else None
if correlation is None:
# A post that is not a benchmark reply is not part of the measured
# workload. Pass it through untraced rather than inventing one.
await self._app(scope, replay, send)
return
self._collector.record(REPLY_RECEIVED, correlation)
token = _in_flight.set(correlation)
status: int | None = None
async def tracing_send(message: dict[str, Any]) -> None:
nonlocal status
if message.get("type") == "http.response.start":
status = int(message["status"])
await send(message)
if (
message.get("type") == "http.response.body"
and not message.get("more_body", False)
and status == 200
):
self._collector.record(REPLY_ACCEPTED, correlation)
try:
await self._app(scope, replay, tracing_send)
finally:
_in_flight.reset(token)
async def _activity(self, scope: Scope, receive: Receive, send: Send) -> None:
body, replay = await _buffer_body(receive)
row = _json_object(body)
await self._app(scope, replay, send)
if (
row is None
or row.get("kind") != "turn"
or row.get("status") not in _TURN_ENDED
):
return
room_id, message_id = row.get("room_id"), row.get("message_id")
if isinstance(room_id, str) and isinstance(message_id, str):
self._collector.record(
TURN_REPORTED,
correlation_for(room_id, message_id),
detail={"status": row["status"], "turn_id": row.get("turn_id")},
)
async def _approval(self, scope: Scope, receive: Receive, send: Send) -> None:
body, replay = await _buffer_body(receive)
request = _json_object(body)
await self._app(scope, replay, send)
if request is None:
return
# A request carries its turn's origin thread, which for a message that
# started no thread is the message itself.
room_id, thread_id = request.get("room_id"), request.get("thread_id")
if isinstance(room_id, str) and isinstance(thread_id, str):
self._collector.record(
APPROVAL_OPENED,
correlation_for(room_id, thread_id),
detail={"request_id": request.get("request_id")},
)
async def _stream(self, scope: Scope, receive: Receive, send: Send) -> None:
async def tracing_send(message: dict[str, Any]) -> None:
if message.get("type") == "http.response.body":
chunk = message.get("body", b"")
if chunk:
for correlation in _sse_correlations(chunk):
self._collector.record(SSE_PUSH, correlation)
await send(message)
await self._app(scope, receive, tracing_send)
class RequestCounter:
"""Every HTTP request the server is sent, counted by route.
A route is named without the ids in its path, so fifty sessions reporting
count as fifty reports rather than fifty different routes. Counted in
front of the application: what a client asked for is the measure,
whatever the server made of it.
"""
def __init__(self, app: Any) -> None:
self._app = app
self.counts: Counter[str] = Counter()
async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None:
if scope.get("type") == "http":
self.counts[route_of(scope.get("path", ""))] += 1
await self._app(scope, receive, send)
def route_of(path: str) -> str:
"""The path with the agent or session id it names left out."""
parts = path.strip("/").split("/")
if len(parts) >= 3 and parts[0] == "agents":
return "/".join([parts[0], *parts[2:]])
if len(parts) >= 3 and parts[0] == "agent-sessions" and parts[1] != "approvals":
return "/".join([parts[0], *parts[2:]])
return "/".join(parts)
def count_statements(engine: AsyncEngine) -> tuple[Counter[str], Callable[[], None]]:
"""Count the statements the server sends its database, for the life of a run.
Everything on the engine, background work included: an idle server's cost
is mostly background work. Returns the running count and a callable that
removes the listener.
"""
counts: Counter[str] = Counter()
def on_execute(*_args: Any) -> None:
counts["statements"] += 1
event.listen(engine.sync_engine, "before_cursor_execute", on_execute)
def remove() -> None:
event.remove(engine.sync_engine, "before_cursor_execute", on_execute)
return counts, remove
def _json_object(body: bytes) -> dict[str, Any] | None:
try:
payload = json.loads(body)
except ValueError:
return None
return payload if isinstance(payload, dict) else None
async def _buffer_body(receive: Receive) -> tuple[bytes, Receive]:
"""Read the whole request body, and return a `receive` that replays it.
The body has to be read here to learn which message the request is about,
and an ASGI `receive` is single-shot, so the application downstream would
otherwise find an empty body.
"""
chunks: list[bytes] = []
messages: list[dict[str, Any]] = []
while True:
message = await receive()
messages.append(message)
if message["type"] != "http.request":
break
chunks.append(message.get("body", b""))
if not message.get("more_body", False):
break
pending = list(messages)
async def replay() -> dict[str, Any]:
if pending:
return pending.pop(0)
return await receive()
return b"".join(chunks), replay
def trace_commits(engine: AsyncEngine, collector: TraceCollector) -> Callable[[], None]:
"""Record the commit of whichever reply is in flight on this task.
Engine level rather than session level: `commit` on the sync engine is the
DBAPI commit itself, which is the moment the reply is stored. Commits from
background work carry no in-flight correlation and are ignored.
Returns a callable that removes the listener.
"""
def on_commit(_conn: Any) -> None:
correlation = _in_flight.get()
if correlation is not None:
collector.record(REPLY_COMMITTED, correlation)
event.listen(engine.sync_engine, "commit", on_commit)
def remove() -> None:
event.remove(engine.sync_engine, "commit", on_commit)
return remove