Skip to content

Open-source AI teammates in Slack, Teams, Discord, Telegram and Mattermost. Self-host it, with any agent harness and any model.

17.5 KB
Raw Blame
"""Correlated trace records for the end-to-end benchmark.

Every point is keyed on one room message: the room it was posted in and the id
Switch minted for it. The points follow that message from the agent's stream
to the session answering it:

``sse_push``
    Switch wrote the room event onto the agent's stream. Server-side.
``provider_dispatch``
    The session host handed the resulting turn to its provider. Host-side,
    reported by the benchmark's scripted provider.
``reply_posted`` / ``reply_failed``
    The provider's ``post_message`` through its session's own MCP server came
    back posted, or the turn gave up on it. Host-side.
``reply_received`` / ``reply_committed`` / ``reply_accepted``
    That ``post_message`` reached Switch (relayed by the watcher), its
    transaction committed, and Switch answered it 200. Server-side.
``turn_reported``
    The session host reported the turn finished as an activity row, which is
    what messaging platforms draw. Server-side.
``approval_requested`` / ``approval_applied``
    The provider asked a person before answering, and was handed the answer.
    Host-side. ``approval_opened`` is Switch recording the request.
``room_connected``
    The provider's ``connect_to_room`` came back; its detail is the result.
    Host-side.

None of those is a production code change: the server points are observed by
wrapping the ASGI app and listening on the engine the harness itself built,
and the host points are emitted by benchmark code.

**Clocks.** Every record carries both a wall reading and a monotonic one.
Within one process, use monotonic — it cannot step. Across the process
boundary (``sse_push`` → ``provider_dispatch`` spans Python and Node) the two
monotonic scales share no origin that is safe to assume, so those spans are
computed from the wall clock, and a drift diagnostic is reported beside them
rather than the caveat being left implicit; see `LatencyReport.cross_process`
and `ClockResidual`, which says what that diagnostic does and does not cover.
"""

from __future__ import annotations

import json
import math
import time
from collections.abc import Container, Iterable, Mapping, Sequence
from dataclasses import asdict, dataclass, field
from pathlib import Path

SSE_PUSH = "sse_push"
PROVIDER_DISPATCH = "provider_dispatch"
REPLY_POSTED = "reply_posted"
REPLY_FAILED = "reply_failed"
APPROVAL_REQUESTED = "approval_requested"
APPROVAL_APPLIED = "approval_applied"
ROOM_CONNECTED = "room_connected"
REPLY_RECEIVED = "reply_received"
REPLY_COMMITTED = "reply_committed"
REPLY_ACCEPTED = "reply_accepted"
TURN_REPORTED = "turn_reported"
APPROVAL_OPENED = "approval_opened"

#: Points recorded inside the Switch server process.
SERVER_POINTS = frozenset(
    {
        SSE_PUSH,
        REPLY_RECEIVED,
        REPLY_COMMITTED,
        REPLY_ACCEPTED,
        TURN_REPORTED,
        APPROVAL_OPENED,
    }
)
#: Points recorded inside a host (Node) process.
HOST_POINTS = frozenset(
    {
        PROVIDER_DISPATCH,
        REPLY_POSTED,
        REPLY_FAILED,
        APPROVAL_REQUESTED,
        APPROVAL_APPLIED,
        ROOM_CONNECTED,
    }
)


def correlation_for(room_id: str, message_id: str) -> str:
    """The identity a room event keeps from stream push to provider dispatch.

    Room id and message id are the only pair carried unchanged across every
    point: the SSE payload has both, the room prompt the provider is handed
    names both, and the reply and turn row the session sends back carry both. A sequence number
    would not do — it is reset by a server restart, which is one of the
    recovery cases the harness exercises.
    """
    return f"{room_id}/{message_id}"


@dataclass(frozen=True, slots=True)
class TraceRecord:
    point: str
    correlation: str
    wall_ns: int
    mono_ns: int
    process: str
    detail: dict[str, object] = field(default_factory=dict)

    @staticmethod
    def now(
        point: str,
        correlation: str,
        *,
        process: str,
        detail: dict[str, object] | None = None,
    ) -> TraceRecord:
        return TraceRecord(
            point=point,
            correlation=correlation,
            wall_ns=time.time_ns(),
            mono_ns=time.monotonic_ns(),
            process=process,
            detail=detail or {},
        )


class TraceCollector:
    """Accumulates trace records from every process in one run.

    Server-side points are appended directly. Host-side points arrive as JSON
    lines written by the benchmark entrypoint, and are merged with `ingest_jsonl`.
    """

    def __init__(self) -> None:
        self._records: list[TraceRecord] = []

    def add(self, record: TraceRecord) -> None:
        self._records.append(record)

    def record(
        self,
        point: str,
        correlation: str,
        *,
        process: str = "switch-core",
        detail: dict[str, object] | None = None,
    ) -> None:
        self.add(TraceRecord.now(point, correlation, process=process, detail=detail))

    def ingest_jsonl(self, path: Path, correlations: Mapping[str, str]) -> int:
        """Merge host-emitted trace records. Returns how many were read.

        A host labels its records with the marker the driver put in the message
        body, because it never learns the id Switch minted for that message;
        `correlations` maps those markers onto the canonical room/message
        correlation the server-side points use.

        A malformed line, or a marker with no mapping, is an error rather than
        something to skip. A silently dropped record becomes a missing span,
        and a missing span becomes a percentile computed over the wrong
        population — which reads as a result rather than as a fault.
        """
        if not path.exists():
            return 0
        text = path.read_text()
        count = 0
        for number, line in enumerate(text.splitlines(), start=1):
            if not line.strip():
                continue
            try:
                raw = json.loads(line)
            except ValueError as exc:
                raise ValueError(
                    f"{path}:{number} is not valid JSON; the host trace is "
                    f"truncated or corrupt and the run cannot be scored: {line!r}"
                ) from exc
            marker = str(raw["correlation"])
            if marker not in correlations:
                raise ValueError(
                    f"{path}:{number} reports {raw['point']} for {marker!r}, which "
                    "the driver never sent. The host dispatched something the "
                    "benchmark did not create; the run cannot be scored."
                )
            self.add(
                TraceRecord(
                    point=str(raw["point"]),
                    correlation=correlations[marker],
                    wall_ns=int(raw["wall_ns"]),
                    mono_ns=int(raw["mono_ns"]),
                    process=str(raw["process"]),
                    detail=dict(raw["detail"]),
                )
            )
            count += 1
        return count

    @property
    def records(self) -> Sequence[TraceRecord]:
        return tuple(self._records)

    def subset(self, correlations: Container[str]) -> TraceCollector:
        """A collector holding only the records for the given correlations.

        One server serves every workload in a run, so its instrumentation keeps
        filling one collector. Scoring a workload against that would average it
        with the ones before it — and since the workloads differ precisely in
        how loaded the server is, that would erase the effect being measured.
        """
        scoped = TraceCollector()
        for record in self._records:
            if record.correlation in correlations:
                scoped.add(record)
        return scoped

    def by_correlation(self) -> dict[str, dict[str, TraceRecord]]:
        """First record of each point, per correlation.

        First, not last: a retried reply is a second `reply_received` for the
        same message, and the latency being measured is the one the room
        actually waited for.

        Keeping only the first is right for latency and wrong as a record of
        what happened, so it is not the only view. Anything that needs to know a
        point occurred more than once must ask `repeats`; this one cannot tell
        it apart from a point that occurred exactly once.
        """
        out: dict[str, dict[str, TraceRecord]] = {}
        for record in self._records:
            points = out.setdefault(record.correlation, {})
            if record.point not in points:
                points[record.point] = record
        return out

    def repeats(self, point: str) -> dict[str, int]:
        """Correlations for which `point` was recorded more than once.

        Maps the correlation to how many times it occurred. Empty is the
        expected answer everywhere the topology is meant to deliver a message
        exactly once.

        This exists because a duplicate is otherwise the one failure this
        harness cannot see. The dispatch waiter is satisfied by the first
        record, ingestion accepts every record it is given, and the latency
        view keeps the first and discards the rest — so a message executed
        twice is scored as a message executed once, with a healthy latency and
        no loss. For work handed to a provider, that is worse than dropping it:
        the run would report a clean result for a topology doing the turn
        twice, which is exactly the regression a change to takeover and
        recovery is most likely to introduce.
        """
        counts: dict[str, int] = {}
        for record in self._records:
            if record.point == point:
                counts[record.correlation] = counts.get(record.correlation, 0) + 1
        return {
            correlation: count
            for correlation, count in sorted(counts.items())
            if count > 1
        }

    def write_jsonl(self, path: Path) -> None:
        path.parent.mkdir(parents=True, exist_ok=True)
        path.write_text(
            "".join(json.dumps(asdict(record)) + "\n" for record in self._records)
        )


def percentile(values: Sequence[float], fraction: float) -> float:
    """Linear-interpolated percentile.

    Raises on an empty population rather than returning zero: a zero here would
    read as "instant" in the metric table, which is the one wrong answer that
    looks like a good result.
    """
    if not values:
        raise ValueError("no samples: a percentile over nothing has no meaning")
    ordered = sorted(values)
    if len(ordered) == 1:
        return ordered[0]
    position = fraction * (len(ordered) - 1)
    low = math.floor(position)
    high = math.ceil(position)
    if low == high:
        return ordered[low]
    return ordered[low] + (ordered[high] - ordered[low]) * (position - low)


@dataclass(frozen=True, slots=True)
class LatencyReport:
    """One latency measure over one run.

    `cross_process` records whether the span was computed from wall-clock
    readings taken in two different processes. When it is true the numbers
    carry the clock-comparison caveat and must be reported with it.
    """

    name: str
    unit: str
    samples: int
    expected: int
    cross_process: bool
    p50: float
    p95: float
    p99: float
    minimum: float
    maximum: float

    @property
    def complete(self) -> bool:
        return self.samples == self.expected

    def as_dict(self) -> dict[str, object]:
        return asdict(self)


def summarise(
    name: str,
    values: Sequence[float],
    *,
    expected: int,
    cross_process: bool,
    unit: str = "ms",
) -> LatencyReport:
    return LatencyReport(
        name=name,
        unit=unit,
        samples=len(values),
        expected=expected,
        cross_process=cross_process,
        p50=percentile(values, 0.50),
        p95=percentile(values, 0.95),
        p99=percentile(values, 0.99),
        minimum=min(values),
        maximum=max(values),
    )


def span_ms(start: TraceRecord, end: TraceRecord) -> float:
    """Elapsed milliseconds between two points.

    Monotonic when both were taken in the same process, wall clock when they
    were not. Mixing the two scales silently is the failure this guards: the
    offset between an arbitrary pair of monotonic origins is unbounded, so a
    cross-process monotonic subtraction can produce a plausible-looking number
    that is wrong by the age of one of the machines' boots.
    """
    if start.process == end.process:
        return (end.mono_ns - start.mono_ns) / 1_000_000
    return (end.wall_ns - start.wall_ns) / 1_000_000


def measure(
    collector: TraceCollector,
    *,
    start_point: str,
    end_point: str,
    name: str,
) -> tuple[LatencyReport, list[str]]:
    """Summarise one span across every correlated message in the run.

    Returns the report and the correlations that could not be measured, so an
    incomplete population is visible in the output instead of quietly
    shrinking the sample count.
    """
    grouped = collector.by_correlation()
    values: list[float] = []
    missing: list[str] = []
    cross_process = False
    for correlation, points in sorted(grouped.items()):
        start = points.get(start_point)
        end = points.get(end_point)
        if start is None or end is None:
            missing.append(correlation)
            continue
        if start.process != end.process:
            cross_process = True
        values.append(span_ms(start, end))
    if not values:
        raise ValueError(
            f"{name}: no correlation had both {start_point} and {end_point}; "
            f"{len(missing)} incomplete. The run produced no measurable span, "
            "which is a harness failure, not a latency of zero."
        )
    return (
        summarise(name, values, expected=len(grouped), cross_process=cross_process),
        missing,
    )


@dataclass(frozen=True, slots=True)
class ClockResidual:
    """How far the wall clock drifted from the monotonic one during a run.

    A cross-process span is computed from wall-clock readings, because that is
    the only scale the two processes share: their monotonic clocks have
    unrelated origins, so subtracting one from the other yields the gap between
    those origins and says nothing about elapsed time.

    One thing that can go wrong with the wall clock is that it steps or slews
    while the run is in progress, and that *is* measurable — within a single
    process, where both scales are available over the same interval. Comparing
    the two elapsed times between that process's first and last record is what
    this reports.

    **It is a diagnostic, not an error bound.** A figure near zero says every
    process saw its own wall clock advance at the rate its monotonic clock did,
    so none of them stepped mid-run. It says nothing about whether two processes
    agreed on what time it was: a constant offset between them drifts not at
    all and would be invisible here, while passing straight into every
    cross-process span as a fixed bias. What limits that offset is that the
    processes share one machine and one system clock, which is an assumption
    about the deployment rather than something these records prove. A run split
    across hosts would need real clock synchronisation before its cross-process
    spans meant anything.
    """

    processes: int
    span_seconds: float
    worst_drift_ms: float


def clock_residual(collector: TraceCollector) -> ClockResidual:
    """Worst wall-versus-monotonic drift observed in any one process."""
    extremes: dict[str, tuple[TraceRecord, TraceRecord]] = {}
    for record in collector.records:
        first, last = extremes.get(record.process, (record, record))
        if record.mono_ns < first.mono_ns:
            first = record
        if record.mono_ns > last.mono_ns:
            last = record
        extremes[record.process] = (first, last)
    drifts: list[float] = []
    longest = 0.0
    for first, last in extremes.values():
        elapsed = (last.mono_ns - first.mono_ns) / 1_000_000_000
        if elapsed <= 0:
            # One record, or several within the clock's resolution: there is no
            # interval to measure drift over, and calling that zero drift would
            # claim evidence the run does not have.
            continue
        longest = max(longest, elapsed)
        wall = (last.wall_ns - first.wall_ns) / 1_000_000
        mono = (last.mono_ns - first.mono_ns) / 1_000_000
        drifts.append(abs(wall - mono))
    if not drifts:
        raise ValueError(
            "no process recorded two points far enough apart to measure clock "
            "drift, so there is no check on whether a wall clock stepped mid-run"
        )
    return ClockResidual(
        processes=len(drifts),
        span_seconds=longest,
        worst_drift_ms=max(drifts),
    )


def format_table(reports: Iterable[LatencyReport]) -> str:
    rows = list(reports)
    if not rows:
        return "(no latency measures)"
    header = f"{'measure':<46} {'n':>6} {'p50':>9} {'p95':>9} {'p99':>9} {'max':>9}"
    lines = [header, "-" * len(header)]
    for report in rows:
        flag = "" if report.complete else f"  ({report.samples}/{report.expected})"
        cross = " *" if report.cross_process else ""
        lines.append(
            f"{report.name + cross:<46} {report.samples:>6} "
            f"{report.p50:>9.2f} {report.p95:>9.2f} {report.p99:>9.2f} "
            f"{report.maximum:>9.2f}{flag}"
        )
    if any(report.cross_process for report in rows):
        lines.append("")
        lines.append(
            "* spans two processes: computed from wall clock, not monotonic "
            "(see clock_residual)"
        )
    return "\n".join(lines)