Open-source AI teammates in Slack, Teams, Discord, Telegram and Mattermost. Self-host it, with any agent harness and any model.
Python agent-orchestrationaiai-agentschatopsdiscordlangchainmcpmicrosoft-teamsmulti-agentopen-sourceopenaiself-hostedslack
"""One benchmark workload: post, wait, and score what it cost.
A workload is a room count, a message count and a way of posting. Everything
else — the server, the host topology, the instrumentation — is identical
between them, so the difference between two workloads' figures is the
difference between the loads.
Three latency measures are reported, and kept apart deliberately.
`sse push → provider dispatch`
Switch wrote the message to the agent's stream; the watcher routed it to
the session placed in the room, over its IPC pipe, and the session host
turned it into a turn for its provider.
`provider dispatch → reply accepted`
The provider called `post_message` on its session's MCP server; the host
sent it up its pipe, and the watcher called Switch as that session.
`sse push → turn reported`
The whole turn as a messaging platform sees it: from the message leaving
Switch to the host's activity row saying the turn finished.
Reported as one number they would average out: an improvement to routing and
an unchanged tool relay would combine into a figure that looked like progress
everywhere. They are different costs with different causes.
"""
from __future__ import annotations
import asyncio
import os
from collections.abc import Callable
from dataclasses import dataclass
from pathlib import Path
from tests.benchmarks.host import BenchWatcher, await_dispatches, marked, new_marker
from tests.benchmarks.metrics import ResourceReport, ResourceSampler
from tests.benchmarks.server import BenchAgent, BenchCore, BenchServer
from tests.benchmarks.trace import (
PROVIDER_DISPATCH,
REPLY_ACCEPTED,
SSE_PUSH,
TURN_REPORTED,
ClockResidual,
LatencyReport,
TraceCollector,
clock_residual,
correlation_for,
format_table,
measure,
)
#: How often the resource sampler reads the process tree and the socket table.
#: Fast enough to catch a session's startup peak, slow enough that `ps` and
#: `lsof` are not themselves a measurable share of the CPU being attributed.
SAMPLE_INTERVAL_SECONDS = 0.5
#: Per-message allowance when waiting for dispatches, on top of a fixed floor.
#: Generous: a timeout here fails the run, and a benchmark that flakes under
#: load reports nothing at all about the load.
SECONDS_PER_MESSAGE = 2.0
DISPATCH_FLOOR_SECONDS = 90.0
#: How long a watcher may take to get its inbound connection registered before
#: the run is called broken. Startup, not steady state, so generous.
STREAM_TIMEOUT_SECONDS = 60.0
def dispatch_timeout(messages: int) -> float:
return DISPATCH_FLOOR_SECONDS + SECONDS_PER_MESSAGE * messages
async def dispatch_wait(
watcher: BenchWatcher, markers: dict[str, str], timeout: float
) -> frozenset[str]:
"""Wait for every dispatch without blocking the loop the server runs on.
The Switch server under measurement is in this process and on this event
loop, so a blocking poll here would stall the thing being measured and
price its own wait into the result.
Returns the correlations that never reached a provider.
"""
return await asyncio.to_thread(await_dispatches, watcher, markers, timeout)
async def await_stream(bench: BenchServer, agent_id: str, timeout: float) -> None:
"""Block until the watcher's inbound connection is registered on the server.
Not politeness, and not a substitute for measuring startup: the watcher's
supervisor exists before its stream does, and a message addressed into that
window is never delivered to it at all — the stream opens at cursor zero and
is advanced to the present rather than replaying what it missed. Posting
before the stream exists therefore measures nothing and loses the message.
Waited on the server's own connection registry, so what is waited for is the
server agreeing the stream is open, not a guess at how long that takes.
"""
loop = asyncio.get_running_loop()
deadline = loop.time() + timeout
while loop.time() < deadline:
if bench.connections.for_agent(agent_id):
return
await asyncio.sleep(0.05)
raise TimeoutError(
f"agent {agent_id} had opened no inbound connection {timeout}s after its "
"watcher supervisor started, so there was nothing to address"
)
def sampler_for(
bench: BenchServer, watcher: BenchWatcher, agent_id: str
) -> ResourceSampler:
return _sampler(
watcher=watcher,
port=bench.port,
# The registry is in this process, so the server's own count of the
# agent's open protocol connections is free to read alongside the
# kernel's count of sockets.
count_streams=lambda: len(bench.connections.for_agent(agent_id)),
)
def sampler_for_core(
core: BenchCore, watcher: BenchWatcher, agent_id: str
) -> ResourceSampler:
"""`sampler_for`, for a scenario that replaces the Core while it samples.
The connection count is read off whichever Core is serving at the moment of
the sample. Bound to one of them instead, every sample after a restart
would be taken from a registry nothing is connected to any more, and the
peak would be whatever the count happened to be when the old Core died.
"""
return _sampler(
watcher=watcher,
# Kept across a restart by design, so reading it once is reading the
# address the hosts are still pointed at.
port=core.server.port,
count_streams=lambda: len(core.server.connections.for_agent(agent_id)),
)
def _sampler(
*, watcher: BenchWatcher, port: int, count_streams: Callable[[], int]
) -> ResourceSampler:
# Both trees: the server runs in this process, and the hosts hang off the
# detached watcher supervisor, which is not a descendant of it.
return ResourceSampler(
root_pids=lambda: [os.getpid(), watcher.supervisor_pid],
port=port,
count_streams=count_streams,
interval_seconds=SAMPLE_INTERVAL_SECONDS,
)
#: The spans reported for every workload, in the order they occur, and whether
#: each needs the turn to have been answered.
SPANS = (
("sse push → provider dispatch", SSE_PUSH, PROVIDER_DISPATCH, False),
("provider dispatch → reply accepted", PROVIDER_DISPATCH, REPLY_ACCEPTED, True),
("sse push → turn reported", SSE_PUSH, TURN_REPORTED, True),
)
#: The points that say a message was answered: its reply taken by Switch, and
#: its turn reported finished.
ANSWERED = (REPLY_ACCEPTED, TURN_REPORTED)
@dataclass(frozen=True, slots=True)
class Posted:
"""The messages one workload sent, and which of them started a session.
`cold` is the first message addressed into each room. That message waits
for a session host to be spawned and its provider started before anything
can take it, so its latency is dominated by process startup — a cost paid once
per session, not once per message. Pooled with the rest it would swamp
them; dropped it would hide the cost of the topology's cold path, which at
fifty sessions is most of what the topology does.
The rest are warm only because they are posted in later rounds, once the
cold round has been served. Posting a room's messages back to back would
put every one of them behind the same process startup and report a warm
figure indistinguishable from the cold one.
`cut` is the messages a fault was aimed at while their turn may have been
running: killed with the process serving them, they must still be
dispatched at most once, but nothing promises they were answered.
"""
markers: dict[str, str]
cold: frozenset[str]
cut: frozenset[str]
async def answer_wait(
collector: TraceCollector, correlations: frozenset[str], timeout: float
) -> frozenset[str]:
"""Wait until each message has been answered; return those that were not.
The points are server-side, recorded on this loop, so this polls the
collector rather than a file.
"""
loop = asyncio.get_running_loop()
deadline = loop.time() + timeout
outstanding = set(correlations)
while True:
grouped = collector.subset(outstanding).by_correlation()
outstanding = {
correlation
for correlation in outstanding
if not all(point in grouped.get(correlation, {}) for point in ANSWERED)
}
if not outstanding or loop.time() >= deadline:
return frozenset(outstanding)
await asyncio.sleep(0.05)
async def post_round(
*,
bench: BenchServer,
poster: BenchAgent,
target: BenchAgent,
room_ids: list[str],
concurrent: bool,
) -> dict[str, str]:
"""Address one marked message into each room, and map marker → correlation.
That map is what lets a host-side record be matched to the server-side
ones: a host is handed command text and never learns the id Switch minted,
so the driver carries the mapping.
`concurrent` decides whether the rooms are addressed one after another or
all at once. Sequential measures a loaded server; concurrent measures the
same messages arriving together, which is the case where a shared inbound
connection and a per-session one differ most.
"""
markers: dict[str, str] = {}
async def address(room_id: str) -> None:
marker = new_marker()
message_id = await bench.address(
sender=poster,
room_id=room_id,
target=target.name,
body=f"@{target.name} {marked(marker)}",
)
markers[marker] = correlation_for(room_id, message_id)
if concurrent:
await asyncio.gather(*(address(room_id) for room_id in room_ids))
else:
for room_id in room_ids:
await address(room_id)
return markers
def score(
*,
label: str,
rooms: int,
collector: TraceCollector,
posted: Posted,
undelivered: frozenset[str],
replies: dict[str, list[str]],
resources: ResourceReport,
) -> WorkloadResult:
"""Turn one workload's traces into the figures that get reported.
`replies` is every stored benchmark reply by the message it answers, with
the rooms it landed in, as `BenchServer.replies` reads them.
Latency is scored over what was delivered, and what was not is counted and
reported beside it. Scoring the undelivered as missing span ends instead
would bury a delivery failure in a measurement complaint; dropping it
without a count would make a workload the topology could not serve look
like one it served quickly.
Delivering a message twice is counted too, and separately. It is the
opposite failure and it does not show up in any of the other figures: an
extra dispatch adds no latency, loses nothing, and leaves the counts of
sent and served messages agreeing with each other. It is counted over
everything posted rather than over what was delivered, because a message
dispatched twice and completed neither time is a duplicate and a delivery
failure at once — scored over the delivered subset it would show only as
the second, which reads as the topology being slow rather than confused.
"""
posted_correlations = set(posted.markers.values())
duplicated = collector.subset(posted_correlations).repeats(PROVIDER_DISPATCH)
replied_twice = collector.subset(posted_correlations).repeats(REPLY_ACCEPTED)
correlations = posted_correlations - undelivered
scoped = collector.subset(correlations)
answerable = correlations - posted.cut
grouped = scoped.by_correlation()
unanswered = {
correlation
for correlation in answerable
if not all(point in grouped.get(correlation, {}) for point in ANSWERED)
}
# A reply in any room but the one it answers was posted where Switch had
# the session placed, and the watcher had it placed somewhere else.
misplaced = {
correlation: rooms
for correlation, rooms in replies.items()
if correlation in posted_correlations
and any(room != correlation.split("/", 1)[0] for room in rooms)
}
latencies: list[LatencyReport] = []
unmeasured: set[str] = set()
for name, start, end, answered in SPANS:
chosen = answerable if answered else correlations
populations = (
("cold", posted.cold & chosen),
("warm", chosen - posted.cold),
)
for suffix, members in populations:
if not members:
continue
report, missing = measure(
scoped.subset(members),
start_point=start,
end_point=end,
name=f"{name} [{suffix}]",
)
latencies.append(report)
unmeasured.update(missing)
return WorkloadResult(
label=label,
rooms=rooms,
messages=len(posted.markers),
undelivered=tuple(sorted(undelivered)),
unanswered=tuple(sorted(unanswered)),
duplicated=tuple(sorted(duplicated.items())),
replied_twice=tuple(sorted(replied_twice.items())),
misplaced=tuple(
sorted((key, tuple(rooms)) for key, rooms in misplaced.items())
),
resources=resources,
latencies=tuple(latencies),
residual=clock_residual(scoped),
unmeasured=tuple(sorted(unmeasured)),
)
@dataclass(frozen=True, slots=True)
class WorkloadResult:
label: str
rooms: int
messages: int
#: Correlations that never reached a provider, named rather than counted so
#: a loss can be traced to the room and message it happened in.
undelivered: tuple[str, ...]
#: Delivered, not cut by a fault, and still without an accepted reply or a
#: finished turn row.
unanswered: tuple[str, ...]
#: Correlations dispatched to a provider more than once, with how many
#: times. A message executed twice is a correctness failure that costs no
#: latency and loses nothing, so it is reported on its own or not at all.
duplicated: tuple[tuple[str, int], ...]
#: Correlations whose reply Switch accepted more than once.
replied_twice: tuple[tuple[str, int], ...]
#: Replies stored in a room other than the one they answer, with the rooms.
misplaced: tuple[tuple[str, tuple[str, ...]], ...]
resources: ResourceReport
latencies: tuple[LatencyReport, ...]
residual: ClockResidual
unmeasured: tuple[str, ...]
async def run_workload(
*,
bench: BenchServer,
watcher: BenchWatcher,
collector: TraceCollector,
poster: BenchAgent,
target: BenchAgent,
room_ids: list[str],
per_room: int,
concurrent: bool,
label: str,
) -> WorkloadResult:
"""Post one workload's messages in rounds, wait for each, and score it.
A round is one message into every room, and the next round is not posted
until this one has been served. That ordering is the only thing separating
the cold path from the warm one: posted back to back, a room's second and
third messages queue behind the very process startup the first is waiting
for, and the warm figures come out equal to the cold ones.
"""
await await_stream(bench, target.agent_id, STREAM_TIMEOUT_SECONDS)
markers: dict[str, str] = {}
cold: dict[str, str] = {}
undelivered: set[str] = set()
sampler = sampler_for(bench, watcher, target.agent_id)
async with sampler.running(label):
for round_index in range(per_room):
sent = await post_round(
bench=bench,
poster=poster,
target=target,
room_ids=room_ids,
concurrent=concurrent,
)
if round_index == 0:
cold = sent
markers.update(sent)
lost = await dispatch_wait(watcher, sent, dispatch_timeout(len(sent)))
undelivered |= lost
await answer_wait(
collector,
frozenset(sent.values()) - lost,
dispatch_timeout(len(sent)),
)
posted = Posted(markers=markers, cold=frozenset(cold.values()), cut=frozenset())
collector.ingest_jsonl(watcher.trace_path, posted.markers)
return score(
label=label,
rooms=len(room_ids),
collector=collector,
posted=posted,
undelivered=frozenset(undelivered),
replies=await bench.replies(),
resources=sampler.last_report,
)
def render(results: list[WorkloadResult]) -> str:
"""The baseline report, as one block of text.
Written for a Slack-bridged room and a terminal alike: one line per item
with its label, no table characters that a chat client will mangle.
"""
out: list[str] = []
for result in results:
resources = result.resources
out.append(f"### {result.label}")
out.append(
f"rooms/sessions {result.rooms} · messages {result.messages} · "
f"never delivered {len(result.undelivered)} · "
f"delivered twice {len(result.duplicated)} · "
f"unanswered {len(result.unanswered)} · "
f"replied twice {len(result.replied_twice)} · "
f"replies in the wrong room {len(result.misplaced)} · "
f"wall {resources.wall_seconds:.1f}s · samples {resources.samples}"
)
out.append(
f"peak agent protocol connections {resources.peak_streams} · "
f"peak TCP sockets to the server {resources.peak_connections} · "
f"peak processes {resources.peak_process_count} · "
f"peak RSS {resources.peak_rss_mib:.0f} MiB · "
f"CPU {resources.cpu_seconds:.1f}s"
)
out.append("")
out.append(format_table(result.latencies))
residual = result.residual
out.append(
f"clock drift: worst wall-vs-monotonic disagreement "
f"{residual.worst_drift_ms:.2f} ms across {residual.processes} "
f"process(es) over {residual.span_seconds:.1f}s"
)
if result.undelivered:
out.append(
f"NOT SERVED: {len(result.undelivered)} of {result.messages} messages "
"never reached a provider; the latency above is over the rest"
)
# Listed, not just counted: a lost message is the kind of finding
# someone has to go and read the room's journals about, and a bare
# count gives them nowhere to start.
out.extend(f" not served: {lost}" for lost in result.undelivered)
if result.duplicated:
out.append(
f"SERVED TWICE: {len(result.duplicated)} of {result.messages} "
"messages were dispatched to a provider more than once; the work "
"was done repeatedly and the latency above is over the first "
"dispatch of each"
)
out.extend(
f" served {count}x: {correlation}"
for correlation, count in result.duplicated
)
out.extend(f" UNANSWERED: {lost}" for lost in result.unanswered)
out.extend(
f" REPLIED {count}x: {correlation}"
for correlation, count in result.replied_twice
)
out.extend(
f" REPLY IN THE WRONG ROOM: {correlation} landed in {', '.join(rooms)}"
for correlation, rooms in result.misplaced
)
if result.unmeasured:
out.append(
f"UNMEASURED: {len(result.unmeasured)} correlations lacked a point; "
"the percentiles above are over a partial population"
)
out.append("")
return "\n".join(out)
def publish(results: list[WorkloadResult], path: Path) -> str:
"""Write the report beside the run and return it for printing."""
text = render(results)
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(text)
return text