Open-source AI teammates in Slack, Teams, Discord, Telegram and Mattermost. Self-host it, with any agent harness and any model.
Python agent-orchestrationaiai-agentschatopsdiscordlangchainmcpmicrosoft-teamsmulti-agentopen-sourceopenaiself-hostedslack
"""Correlated trace records for the end-to-end benchmark.
Every point is keyed on one room message: the room it was posted in and the id
Switch minted for it. The points follow that message from the agent's stream
to the session answering it:
``sse_push``
Switch wrote the room event onto the agent's stream. Server-side.
``provider_dispatch``
The session host handed the resulting turn to its provider. Host-side,
reported by the benchmark's scripted provider.
``reply_posted`` / ``reply_failed``
The provider's ``post_message`` through its session's own MCP server came
back posted, or the turn gave up on it. Host-side.
``reply_received`` / ``reply_committed`` / ``reply_accepted``
That ``post_message`` reached Switch (relayed by the watcher), its
transaction committed, and Switch answered it 200. Server-side.
``turn_reported``
The session host reported the turn finished as an activity row, which is
what messaging platforms draw. Server-side.
``approval_requested`` / ``approval_applied``
The provider asked a person before answering, and was handed the answer.
Host-side. ``approval_opened`` is Switch recording the request.
``room_connected``
The provider's ``connect_to_room`` came back; its detail is the result.
Host-side.
None of those is a production code change: the server points are observed by
wrapping the ASGI app and listening on the engine the harness itself built,
and the host points are emitted by benchmark code.
**Clocks.** Every record carries both a wall reading and a monotonic one.
Within one process, use monotonic — it cannot step. Across the process
boundary (``sse_push`` → ``provider_dispatch`` spans Python and Node) the two
monotonic scales share no origin that is safe to assume, so those spans are
computed from the wall clock, and a drift diagnostic is reported beside them
rather than the caveat being left implicit; see `LatencyReport.cross_process`
and `ClockResidual`, which says what that diagnostic does and does not cover.
"""
from __future__ import annotations
import json
import math
import time
from collections.abc import Container, Iterable, Mapping, Sequence
from dataclasses import asdict, dataclass, field
from pathlib import Path
SSE_PUSH = "sse_push"
PROVIDER_DISPATCH = "provider_dispatch"
REPLY_POSTED = "reply_posted"
REPLY_FAILED = "reply_failed"
APPROVAL_REQUESTED = "approval_requested"
APPROVAL_APPLIED = "approval_applied"
ROOM_CONNECTED = "room_connected"
REPLY_RECEIVED = "reply_received"
REPLY_COMMITTED = "reply_committed"
REPLY_ACCEPTED = "reply_accepted"
TURN_REPORTED = "turn_reported"
APPROVAL_OPENED = "approval_opened"
#: Points recorded inside the Switch server process.
SERVER_POINTS = frozenset(
{
SSE_PUSH,
REPLY_RECEIVED,
REPLY_COMMITTED,
REPLY_ACCEPTED,
TURN_REPORTED,
APPROVAL_OPENED,
}
)
#: Points recorded inside a host (Node) process.
HOST_POINTS = frozenset(
{
PROVIDER_DISPATCH,
REPLY_POSTED,
REPLY_FAILED,
APPROVAL_REQUESTED,
APPROVAL_APPLIED,
ROOM_CONNECTED,
}
)
def correlation_for(room_id: str, message_id: str) -> str:
"""The identity a room event keeps from stream push to provider dispatch.
Room id and message id are the only pair carried unchanged across every
point: the SSE payload has both, the room prompt the provider is handed
names both, and the reply and turn row the session sends back carry both. A sequence number
would not do — it is reset by a server restart, which is one of the
recovery cases the harness exercises.
"""
return f"{room_id}/{message_id}"
@dataclass(frozen=True, slots=True)
class TraceRecord:
point: str
correlation: str
wall_ns: int
mono_ns: int
process: str
detail: dict[str, object] = field(default_factory=dict)
@staticmethod
def now(
point: str,
correlation: str,
*,
process: str,
detail: dict[str, object] | None = None,
) -> TraceRecord:
return TraceRecord(
point=point,
correlation=correlation,
wall_ns=time.time_ns(),
mono_ns=time.monotonic_ns(),
process=process,
detail=detail or {},
)
class TraceCollector:
"""Accumulates trace records from every process in one run.
Server-side points are appended directly. Host-side points arrive as JSON
lines written by the benchmark entrypoint, and are merged with `ingest_jsonl`.
"""
def __init__(self) -> None:
self._records: list[TraceRecord] = []
def add(self, record: TraceRecord) -> None:
self._records.append(record)
def record(
self,
point: str,
correlation: str,
*,
process: str = "switch-core",
detail: dict[str, object] | None = None,
) -> None:
self.add(TraceRecord.now(point, correlation, process=process, detail=detail))
def ingest_jsonl(self, path: Path, correlations: Mapping[str, str]) -> int:
"""Merge host-emitted trace records. Returns how many were read.
A host labels its records with the marker the driver put in the message
body, because it never learns the id Switch minted for that message;
`correlations` maps those markers onto the canonical room/message
correlation the server-side points use.
A malformed line, or a marker with no mapping, is an error rather than
something to skip. A silently dropped record becomes a missing span,
and a missing span becomes a percentile computed over the wrong
population — which reads as a result rather than as a fault.
"""
if not path.exists():
return 0
text = path.read_text()
count = 0
for number, line in enumerate(text.splitlines(), start=1):
if not line.strip():
continue
try:
raw = json.loads(line)
except ValueError as exc:
raise ValueError(
f"{path}:{number} is not valid JSON; the host trace is "
f"truncated or corrupt and the run cannot be scored: {line!r}"
) from exc
marker = str(raw["correlation"])
if marker not in correlations:
raise ValueError(
f"{path}:{number} reports {raw['point']} for {marker!r}, which "
"the driver never sent. The host dispatched something the "
"benchmark did not create; the run cannot be scored."
)
self.add(
TraceRecord(
point=str(raw["point"]),
correlation=correlations[marker],
wall_ns=int(raw["wall_ns"]),
mono_ns=int(raw["mono_ns"]),
process=str(raw["process"]),
detail=dict(raw["detail"]),
)
)
count += 1
return count
@property
def records(self) -> Sequence[TraceRecord]:
return tuple(self._records)
def subset(self, correlations: Container[str]) -> TraceCollector:
"""A collector holding only the records for the given correlations.
One server serves every workload in a run, so its instrumentation keeps
filling one collector. Scoring a workload against that would average it
with the ones before it — and since the workloads differ precisely in
how loaded the server is, that would erase the effect being measured.
"""
scoped = TraceCollector()
for record in self._records:
if record.correlation in correlations:
scoped.add(record)
return scoped
def by_correlation(self) -> dict[str, dict[str, TraceRecord]]:
"""First record of each point, per correlation.
First, not last: a retried reply is a second `reply_received` for the
same message, and the latency being measured is the one the room
actually waited for.
Keeping only the first is right for latency and wrong as a record of
what happened, so it is not the only view. Anything that needs to know a
point occurred more than once must ask `repeats`; this one cannot tell
it apart from a point that occurred exactly once.
"""
out: dict[str, dict[str, TraceRecord]] = {}
for record in self._records:
points = out.setdefault(record.correlation, {})
if record.point not in points:
points[record.point] = record
return out
def repeats(self, point: str) -> dict[str, int]:
"""Correlations for which `point` was recorded more than once.
Maps the correlation to how many times it occurred. Empty is the
expected answer everywhere the topology is meant to deliver a message
exactly once.
This exists because a duplicate is otherwise the one failure this
harness cannot see. The dispatch waiter is satisfied by the first
record, ingestion accepts every record it is given, and the latency
view keeps the first and discards the rest — so a message executed
twice is scored as a message executed once, with a healthy latency and
no loss. For work handed to a provider, that is worse than dropping it:
the run would report a clean result for a topology doing the turn
twice, which is exactly the regression a change to takeover and
recovery is most likely to introduce.
"""
counts: dict[str, int] = {}
for record in self._records:
if record.point == point:
counts[record.correlation] = counts.get(record.correlation, 0) + 1
return {
correlation: count
for correlation, count in sorted(counts.items())
if count > 1
}
def write_jsonl(self, path: Path) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(
"".join(json.dumps(asdict(record)) + "\n" for record in self._records)
)
def percentile(values: Sequence[float], fraction: float) -> float:
"""Linear-interpolated percentile.
Raises on an empty population rather than returning zero: a zero here would
read as "instant" in the metric table, which is the one wrong answer that
looks like a good result.
"""
if not values:
raise ValueError("no samples: a percentile over nothing has no meaning")
ordered = sorted(values)
if len(ordered) == 1:
return ordered[0]
position = fraction * (len(ordered) - 1)
low = math.floor(position)
high = math.ceil(position)
if low == high:
return ordered[low]
return ordered[low] + (ordered[high] - ordered[low]) * (position - low)
@dataclass(frozen=True, slots=True)
class LatencyReport:
"""One latency measure over one run.
`cross_process` records whether the span was computed from wall-clock
readings taken in two different processes. When it is true the numbers
carry the clock-comparison caveat and must be reported with it.
"""
name: str
unit: str
samples: int
expected: int
cross_process: bool
p50: float
p95: float
p99: float
minimum: float
maximum: float
@property
def complete(self) -> bool:
return self.samples == self.expected
def as_dict(self) -> dict[str, object]:
return asdict(self)
def summarise(
name: str,
values: Sequence[float],
*,
expected: int,
cross_process: bool,
unit: str = "ms",
) -> LatencyReport:
return LatencyReport(
name=name,
unit=unit,
samples=len(values),
expected=expected,
cross_process=cross_process,
p50=percentile(values, 0.50),
p95=percentile(values, 0.95),
p99=percentile(values, 0.99),
minimum=min(values),
maximum=max(values),
)
def span_ms(start: TraceRecord, end: TraceRecord) -> float:
"""Elapsed milliseconds between two points.
Monotonic when both were taken in the same process, wall clock when they
were not. Mixing the two scales silently is the failure this guards: the
offset between an arbitrary pair of monotonic origins is unbounded, so a
cross-process monotonic subtraction can produce a plausible-looking number
that is wrong by the age of one of the machines' boots.
"""
if start.process == end.process:
return (end.mono_ns - start.mono_ns) / 1_000_000
return (end.wall_ns - start.wall_ns) / 1_000_000
def measure(
collector: TraceCollector,
*,
start_point: str,
end_point: str,
name: str,
) -> tuple[LatencyReport, list[str]]:
"""Summarise one span across every correlated message in the run.
Returns the report and the correlations that could not be measured, so an
incomplete population is visible in the output instead of quietly
shrinking the sample count.
"""
grouped = collector.by_correlation()
values: list[float] = []
missing: list[str] = []
cross_process = False
for correlation, points in sorted(grouped.items()):
start = points.get(start_point)
end = points.get(end_point)
if start is None or end is None:
missing.append(correlation)
continue
if start.process != end.process:
cross_process = True
values.append(span_ms(start, end))
if not values:
raise ValueError(
f"{name}: no correlation had both {start_point} and {end_point}; "
f"{len(missing)} incomplete. The run produced no measurable span, "
"which is a harness failure, not a latency of zero."
)
return (
summarise(name, values, expected=len(grouped), cross_process=cross_process),
missing,
)
@dataclass(frozen=True, slots=True)
class ClockResidual:
"""How far the wall clock drifted from the monotonic one during a run.
A cross-process span is computed from wall-clock readings, because that is
the only scale the two processes share: their monotonic clocks have
unrelated origins, so subtracting one from the other yields the gap between
those origins and says nothing about elapsed time.
One thing that can go wrong with the wall clock is that it steps or slews
while the run is in progress, and that *is* measurable — within a single
process, where both scales are available over the same interval. Comparing
the two elapsed times between that process's first and last record is what
this reports.
**It is a diagnostic, not an error bound.** A figure near zero says every
process saw its own wall clock advance at the rate its monotonic clock did,
so none of them stepped mid-run. It says nothing about whether two processes
agreed on what time it was: a constant offset between them drifts not at
all and would be invisible here, while passing straight into every
cross-process span as a fixed bias. What limits that offset is that the
processes share one machine and one system clock, which is an assumption
about the deployment rather than something these records prove. A run split
across hosts would need real clock synchronisation before its cross-process
spans meant anything.
"""
processes: int
span_seconds: float
worst_drift_ms: float
def clock_residual(collector: TraceCollector) -> ClockResidual:
"""Worst wall-versus-monotonic drift observed in any one process."""
extremes: dict[str, tuple[TraceRecord, TraceRecord]] = {}
for record in collector.records:
first, last = extremes.get(record.process, (record, record))
if record.mono_ns < first.mono_ns:
first = record
if record.mono_ns > last.mono_ns:
last = record
extremes[record.process] = (first, last)
drifts: list[float] = []
longest = 0.0
for first, last in extremes.values():
elapsed = (last.mono_ns - first.mono_ns) / 1_000_000_000
if elapsed <= 0:
# One record, or several within the clock's resolution: there is no
# interval to measure drift over, and calling that zero drift would
# claim evidence the run does not have.
continue
longest = max(longest, elapsed)
wall = (last.wall_ns - first.wall_ns) / 1_000_000
mono = (last.mono_ns - first.mono_ns) / 1_000_000
drifts.append(abs(wall - mono))
if not drifts:
raise ValueError(
"no process recorded two points far enough apart to measure clock "
"drift, so there is no check on whether a wall clock stepped mid-run"
)
return ClockResidual(
processes=len(drifts),
span_seconds=longest,
worst_drift_ms=max(drifts),
)
def format_table(reports: Iterable[LatencyReport]) -> str:
rows = list(reports)
if not rows:
return "(no latency measures)"
header = f"{'measure':<46} {'n':>6} {'p50':>9} {'p95':>9} {'p99':>9} {'max':>9}"
lines = [header, "-" * len(header)]
for report in rows:
flag = "" if report.complete else f" ({report.samples}/{report.expected})"
cross = " *" if report.cross_process else ""
lines.append(
f"{report.name + cross:<46} {report.samples:>6} "
f"{report.p50:>9.2f} {report.p95:>9.2f} {report.p99:>9.2f} "
f"{report.maximum:>9.2f}{flag}"
)
if any(report.cross_process for report in rows):
lines.append("")
lines.append(
"* spans two processes: computed from wall clock, not monotonic "
"(see clock_residual)"
)
return "\n".join(lines)