Open-source AI teammates in Slack, Teams, Discord, Telegram and Mattermost. Self-host it, with any agent harness and any model.
Python agent-orchestrationaiai-agentschatopsdiscordlangchainmcpmicrosoft-teamsmulti-agentopen-sourceopenaiself-hostedslack
"""Turning a run's raw records into numbers per phase, and into summary.md."""
from __future__ import annotations
import math
import statistics
from collections import defaultdict
def pct(xs: list[float], p: float) -> float | None:
"""The p-th quantile (nearest rank) in milliseconds, or None."""
if not xs:
return None
xs = sorted(xs)
# The ceil(p * n)-th value, rounded first: 0.07 * 100 is 7.000000000000001.
rank = math.ceil(round(p * len(xs), 9))
return round(xs[min(len(xs), max(rank, 1)) - 1] * 1000, 1)
def _avg(xs: list[float]) -> float | None:
return round(statistics.mean(xs), 1) if xs else None
def _max(xs: list[float]) -> float | None:
return round(max(xs), 1) if xs else None
def phase_summary(
phase: dict, stats, samples: list[dict], agents_connected: list[int]
) -> dict:
"""The numbers for one measured window (`phase` has its name, scenario,
length in seconds and the target agent count)."""
name, secs = phase["name"], phase["seconds"]
msgs = stats.by_phase.get(name, [])
ok = [m for m in msgs if m.status == 200]
lat = [x for m in ok for x in m.latencies]
send = [m.acked - m.sent for m in ok if m.acked is not None]
expected = sum(m.expected for m in ok)
got = sum(min(len(m.latencies), m.expected) for m in ok)
ev = stats.events_by_phase.get(name, [])
count = defaultdict(int)
for e in ev:
count[e["kind"]] += 1
# Tagged with the phase as each line arrives, so pod clocks do not matter.
in_window = [s for s in samples if s.get("phase") == name]
pods: dict[str, dict] = {}
for s in in_window:
if s["kind"] != "pod":
continue
p = pods.setdefault(s["pod"], {"cpu": [], "mem": []})
if s.get("cpu_pct") is not None:
p["cpu"].append(s["cpu_pct"])
if s.get("mem_mb") is not None:
p["mem"].append(s["mem_mb"])
db = [s for s in in_window if s["kind"] == "db"]
xact = [s["xact_s"] for s in db if s.get("xact_s") is not None]
errors = (
stats.send_errors.get(name, 0)
+ stats.read_errors.get(name, 0)
+ count["refused"]
+ count["drop"]
+ count["controller_drop"]
+ count["connect_failed"]
+ count["evicted"]
)
return {
"phase": name + (" (stopped early)" if phase.get("stopped") else ""),
"scenario": phase["scenario"],
"agents_target": phase["agents"],
"agents_connected_min": min(agents_connected) if agents_connected else None,
"agents_connected_end": agents_connected[-1] if agents_connected else None,
"seconds": round(secs, 1),
"schedule_fingerprint": phase.get("fingerprint"),
"messages_planned": phase.get("planned", 0),
"messages_sent": len(ok),
"messages_sent_per_s": round(len(ok) / secs, 2) if secs else None,
"send_errors": stats.send_errors.get(name, 0),
"send_ms_p50": pct(send, 0.5),
"send_ms_p99": pct(send, 0.99),
"deliveries_expected": expected,
"deliveries_received": got,
"deliveries_missing": expected - got,
"deliveries_per_s": round(got / secs, 2) if secs else None,
"delivery_ms_p50": pct(lat, 0.5),
"delivery_ms_p99": pct(lat, 0.99),
"delivery_ms_max": pct(lat, 1.0),
"context_reads": len(stats.reads.get(name, [])),
"context_read_errors": stats.read_errors.get(name, 0),
"context_read_ms_p99": pct(stats.reads.get(name, []), 0.99),
"drops": count["drop"],
"refusals": count["refused"],
"connect_failures": count["connect_failed"],
"evictions": count["evicted"],
"gaps": count["gap"],
# Agents that gave up for good (`--retry runtime`, refused as taken
# over), and controller sockets that dropped or were refused.
"stood_down": count["stood_down"],
"controller_drops": count["controller_drop"],
# Agents a dropped controller socket took offline with it.
"detached": count["detached"],
"controller_connect_failures": count["controller_connect_failed"],
"errors": errors,
"harness_lag_ms_max": round(stats.lag_max.get(name, 0.0) * 1000, 1),
"pods": {
pod: {
"cpu_pct_avg": _avg(v["cpu"]),
"cpu_pct_max": _max(v["cpu"]),
"mem_mb_max": _max(v["mem"]),
}
for pod, v in sorted(pods.items())
},
"db_app_conns_max": max((s["app_total"] for s in db), default=None),
"db_app_active_max": max((s["app_active"] for s in db), default=None),
"db_app_idle_tx_max": max((s["app_idle_tx"] for s in db), default=None),
"db_conns_total_max": max((s["db_total"] for s in db), default=None),
"db_lock_waits_max": max((s["lock_waits"] for s in db), default=None),
"db_xact_per_s_avg": _avg(xact),
"db_xact_per_s_p50": round(statistics.median(xact), 1) if xact else None,
"db_xact_per_s_max": _max(xact),
"rds": phase.get("rds"),
}
def _fmt(v) -> str:
return "-" if v is None else str(v)
def render(meta: dict, phases: list[dict], teardown: dict | None) -> str:
lines = [
f"# Load run {meta['run_id']}",
"",
f"- Target: namespace `{meta['namespace']}` "
f"(context `{meta['kube_context'] or 'in-cluster'}`), "
f"API {meta['api_url']}, via {meta['via']}",
f"- switch-core pods seen: {', '.join(meta['pods']) or 'none'}",
f"- Seed {meta['seed']}, membership {meta['membership_fingerprint']}, "
f"{meta['rooms']} rooms of {meta['room_size']}",
f"- Started {meta['started']}, ended {meta.get('ended', '-')}",
"",
"Delivery latency is from the moment the sender's POST went out to the moment the "
"receiving agent read the frame, both on the harness clock. `errors` counts failed "
"sends, failed context reads, refused or failed connects, evictions and dropped sockets.",
"",
"## Traffic",
"",
"| phase | agents (target / min connected) | msgs/s sent | deliveries/s | delivered / expected "
"| delivery p50 / p99 ms | send p99 ms | errors | harness lag max ms |",
"|---|---|---|---|---|---|---|---|---|",
]
for p in phases:
lines.append(
f"| {p['phase']} | {p['agents_target']} / {_fmt(p['agents_connected_min'])} "
f"| {_fmt(p['messages_sent_per_s'])} | {_fmt(p['deliveries_per_s'])} "
f"| {p['deliveries_received']} / {p['deliveries_expected']} "
f"| {_fmt(p['delivery_ms_p50'])} / {_fmt(p['delivery_ms_p99'])} "
f"| {_fmt(p['send_ms_p99'])} | {p['errors']} | {p['harness_lag_ms_max']} |"
)
lines += [
"",
"## Server",
"",
"| phase | pod | CPU avg / max % | memory max MB |",
"|---|---|---|---|",
]
for p in phases:
for pod, v in p["pods"].items():
lines.append(
f"| {p['phase']} | {pod} | {_fmt(v['cpu_pct_avg'])} / {_fmt(v['cpu_pct_max'])} "
f"| {_fmt(v['mem_mb_max'])} |"
)
lines += [
"",
"| phase | DB conns app max (active / idle in tx) | DB conns total max | transactions/s avg / p50 / max "
"| lock waits max | RDS CPU avg / max % |",
"|---|---|---|---|---|---|",
]
for p in phases:
rds = p.get("rds") or {}
lines.append(
f"| {p['phase']} | {_fmt(p['db_app_conns_max'])} ({_fmt(p['db_app_active_max'])} / "
f"{_fmt(p['db_app_idle_tx_max'])}) | {_fmt(p['db_conns_total_max'])} "
f"| {_fmt(p['db_xact_per_s_avg'])} / {_fmt(p['db_xact_per_s_p50'])} / {_fmt(p['db_xact_per_s_max'])} "
f"| {_fmt(p['db_lock_waits_max'])} | {_fmt(rds.get('cpu_avg'))} / {_fmt(rds.get('cpu_max'))} |"
)
if teardown is not None:
lines += [
"",
"## Teardown",
"",
]
if meta.get("kept"):
lines.append(
f"Kept the agents and rooms for --reuse {meta['run_id']}; revoked "
f"{teardown.get('controllers_revoked')} controllers, errors {teardown.get('errors')}."
)
return "\n".join(lines) + "\n"
lines += [
f"Agents deleted {teardown.get('agents_deleted')}, rooms archived {teardown.get('rooms_archived')}, "
f"keys deleted {teardown.get('keys_deleted')}, errors {teardown.get('errors')}. "
f"Left behind: {teardown.get('left_agents')} agents, {teardown.get('left_rooms')} active rooms, "
f"{teardown.get('left_keys')} keys.",
]
return "\n".join(lines) + "\n"