Open-source AI teammates in Slack, Teams, Discord, Telegram and Mattermost. Self-host it, with any agent harness and any model.
Python agent-orchestrationaiai-agentschatopsdiscordlangchainmcpmicrosoft-teamsmulti-agentopen-sourceopenaiself-hostedslack
"""Runs the real Node host topology against the benchmark's own server.
What the connection model costs is decided almost entirely on this side, so
the benchmark runs the shipped processes: a detached watcher supervisor, the
watcher it keeps running (which holds the agent's one connection, owns the
room → session map and makes every Switch call), and one session host per
session, each a child of the watcher talking to it over an IPC pipe and
serving the Switch MCP tools to its provider on loopback. The only
substitution is the provider, a scripted one, made in the benchmark's own
entrypoint under `console/packages/agent-providers/src/host/bench/`.
Two isolations matter, and both are load-bearing rather than tidiness:
`HOME` is redirected
Session state lives under `$HOME/.local/state/switch`, and a watcher
restarts sessions it finds there, so a benchmark watcher sharing a home
directory with a real Switch Console could reach that user's sessions.
the agent is registered per run
A watcher acts on its agent id. A fresh id per run means a benchmark can
never reach a session it did not create, whatever the state on disk.
"""
from __future__ import annotations
import hashlib
import json
import os
import shutil
import signal
import subprocess
import time
import uuid
from collections.abc import Callable, Iterator, Mapping
from contextlib import contextmanager
from dataclasses import dataclass
from pathlib import Path
from typing import Any
from tests.benchmarks.metrics import descendants
from tests.benchmarks.trace import PROVIDER_DISPATCH
#: Prefix of the token the driver embeds in each message body, matched by
#: `benchMarker` in the benchmark adapter. The two must agree.
MARKER_PREFIX = "switch-bench:"
_REPO_ROOT = Path(__file__).resolve().parents[3]
_CONSOLE = _REPO_ROOT / "console"
_BUNDLE = (
_CONSOLE / "packages" / "agent-providers" / "dist-bench" / "bench-host-daemon.mjs"
)
#: What a benchmark session says it can be asked to do. The scripted provider
#: asks for approvals when a message tells it to, so those are on; it answers
#: no questions and cannot be interrupted, compacted or moved between models.
SESSION_CAPABILITIES = {
"input": "queue",
"approvals": True,
"questions": False,
"interrupt": False,
"reset": True,
"compact": False,
"modelChange": False,
"attachmentMimeTypes": [],
}
#: Namespace Switch Console derives a controller's connection id under. It is
#: mirrored here rather than read from the app because the watcher takes the id
#: from the template it is handed, so a harness that minted its own would test a
#: connection identity no Console ever opens.
CONTROLLER_CONNECTION_NAMESPACE = uuid.UUID("c3f2b0de-2e5a-5a1e-9d4a-1f7c2a6b8e05")
def controller_connection_id(agent_id: str) -> str:
"""The connection a controller for this agent opens, as Console derives it.
Derived from the agent rather than minted, so two controllers started for
one agent collide on the server and one takes the connection from the
other, and so a controller that is restarted reopens the connection it had
instead of leaving a second one to be swept.
"""
return str(uuid.uuid5(CONTROLLER_CONNECTION_NAMESPACE, agent_id))
def new_marker() -> str:
"""A token identifying one benchmark message, as the host will report it."""
return str(uuid.uuid4())
def marked(marker: str) -> str:
"""The marker as it must appear in a message body for a host to find it."""
return f"{MARKER_PREFIX}{marker}"
def build_bench_bundle() -> Path:
"""Bundle the benchmark host entrypoint, returning the bundle path.
Bundled rather than run from source because a host is a child process and
the entrypoint imports its package by extensionless specifier, which Node's
own TypeScript support does not resolve. Built every run so a benchmark can
never score a revision against a bundle left over from another one.
"""
if not _CONSOLE.is_dir():
raise RuntimeError(
f"{_CONSOLE} is missing; the benchmark needs the Console workspace to "
"build its host entrypoint"
)
result = subprocess.run(
[
"pnpm",
"--filter",
"@switch-console/agent-providers",
"exec",
"tsdown",
"--config",
"tsdown.bench.config.ts",
],
cwd=_CONSOLE,
capture_output=True,
text=True,
timeout=600,
)
if result.returncode != 0:
raise RuntimeError(
"building the benchmark host entrypoint failed. Run `pnpm install` in "
f"{_CONSOLE} first.\n{result.stdout}\n{result.stderr}"
)
if not _BUNDLE.is_file():
raise RuntimeError(f"the bundler reported success but {_BUNDLE} is not there")
return _BUNDLE
def successor_bundle(bundle: Path) -> Path:
"""The same build again under a second identity, for an upgrade to it.
A bench host names the path it was started from as its build, so one bundle
at two paths is two builds running identical code — which is what an
upgrade between consecutive releases of one topology does to a machine, and
it needs no second checkout to stage.
The copy goes beside the original because the bundle resolves its
dependencies from the workspace it was built in; anywhere else it fails to
load rather than running as an older build would.
"""
successor = bundle.with_name(f"{bundle.stem}-successor{bundle.suffix}")
shutil.copy2(bundle, successor)
return successor
@dataclass(slots=True)
class BenchWatcher:
"""A running watcher and everything spawned beneath it.
`supervisor_pid` is not fixed for the life of the run: a controller can be
killed and another started on the same state, and everything that reads the
process tree has to follow it to the one that is running now.
"""
root: Path
home: Path
trace_path: Path
bundle: Path
connection_id: str
template_path: Path
environment: Mapping[str, str]
supervisor_pid: int
def process_tree(self) -> list[int]:
return descendants(self.supervisor_pid)
def failure(self) -> str | None:
"""The message a host or the watcher wrote before dying, if one did.
Their output goes to log files, so a failure is otherwise visible only
as a workload that never completes.
"""
for path in sorted(self.home.rglob("failure.json")):
return str(json.loads(path.read_text())["message"])
return None
def taken_over(self) -> dict[str, str] | None:
"""The record this controller left if another took its connection.
Written when the server evicts it in favour of a second controller for
the same agent, and kept: standing down is durable, so a controller
that has this file is one that will not reopen the connection on its
own.
"""
record = self.root / "taken-over.json"
if not record.exists():
return None
return {str(k): str(v) for k, v in json.loads(record.read_text()).items()}
def controller_running(self) -> bool:
"""Whether this controller's supervisor is still up."""
return alive(self.supervisor_pid)
def worker_pid(self) -> int | None:
"""The watcher process the supervisor keeps running, if one is alive.
It is the one holding the agent's connection and the parent of every
session host, and it records itself in the lock it holds on the root.
"""
return _owner(self.root / "shared-owner.lock")
def sessions_by_room(self) -> dict[str, str]:
"""Which session the watcher has placed in each room.
Read from `placements.json`, the map the watcher routes by and states
to Switch, rather than inferred from what was delivered where.
"""
path = self.root / "placements.json"
if not path.exists():
return {}
placed = json.loads(path.read_text())["placements"]
return {str(room): str(session) for session, room in placed.items()}
def assigned_sessions(self) -> set[str]:
"""Every session this watcher ever started for a room, placed or not."""
journal = self.root / "assignments.jsonl"
if not journal.exists():
return set()
assigned: set[str] = set()
for line in journal.read_text().splitlines():
if not line.strip():
continue
record = json.loads(line)
# The journal also carries sequence bookkeeping — a numbering
# restart, a routed sequence, a held delivery and its release —
# and none of those name a session.
if "config" in record:
assigned.add(record["config"]["session"]["sessionId"])
return assigned
def session_root(self, session_id: str) -> Path:
"""Where a session of this watcher keeps its state on disk."""
digest = hashlib.sha256(session_id.encode()).hexdigest()
return self.home / ".local" / "state" / "switch" / "sdk-sessions" / digest
def session_pid(self, session_id: str) -> int | None:
"""The session host's process, if one is running for this session."""
return _owner(self.session_root(session_id) / "shared-owner.lock")
def session_pids(self) -> set[int]:
"""Every session host running for a session this watcher started."""
return {
pid
for session_id in self.assigned_sessions()
if (pid := self.session_pid(session_id)) is not None
}
def parked(self, session_id: str) -> bool:
"""Whether the session's host last stopped by parking itself."""
state = self.session_root(session_id) / "shared-state.jsonl"
if not state.exists():
return False
last = None
for line in state.read_text().splitlines():
if line.strip():
kind = json.loads(line)["type"]
if kind in ("running", "parked"):
last = kind
return last == "parked"
def provider_conversations(self, session_id: str) -> list[str]:
"""The provider conversations this session has run, in order.
One entry per time the provider named its conversation, which a start
does both by announcing and by returning, so a single start can appear
twice. What the identity says is what matters: a host that recovered a
session resumes the identity it had, so an entry differing from the
first is a session that came back as a new conversation rather than
the one it was.
"""
inbox = self.session_root(session_id) / "inbox.jsonl"
if not inbox.exists():
raise RuntimeError(f"session {session_id} has no inbox at {inbox}")
native: list[str] = []
for line in inbox.read_text().splitlines():
if not line.strip():
continue
record = json.loads(line)
if record.get("type") == "native":
native.append(record["nativeSessionId"])
return native
def kill_session(self, session_id: str) -> int:
"""Kill a session's host outright, returning its pid.
SIGKILL, not SIGTERM: the case being measured is a host lost without
warning, which writes nothing on the way out. The watcher that is its
parent survives and is what has to bring it back.
"""
pid = self.session_pid(session_id)
if pid is None:
raise RuntimeError(f"session {session_id} has no running host to kill")
os.kill(pid, signal.SIGKILL)
return pid
def kill_worker(self) -> int:
"""Kill the watcher process, leaving the supervisor that keeps it running.
The session hosts are its children and go with it; the supervisor is
what brings the watcher back, and the watcher what brings them back.
SIGKILL for the same reason as a lost host.
"""
pid = self.worker_pid()
if pid is None:
raise RuntimeError(f"no watcher process holds {self.root}")
os.kill(pid, signal.SIGKILL)
return pid
def await_sessions_gone(self, pids: set[int], timeout: float) -> None:
"""Until every one of these session hosts has exited.
A session host is a child of its watcher and ends when its pipe to it
closes. One that outlived its watcher would be a session nothing can
reach, still holding its provider.
"""
deadline = time.monotonic() + timeout
while any(alive(pid) for pid in pids):
if time.monotonic() >= deadline:
survivors = sorted(pid for pid in pids if alive(pid))
raise RuntimeError(
f"session host(s) {survivors} outlived their watcher by {timeout}s"
)
time.sleep(0.05)
def stop_controller(self) -> set[int]:
"""Kill the controller outright, as a Console that is killed is lost.
The supervisor and the watcher are sent SIGKILL, so nothing is written
on the way out: the state a restart has to recover from is a stale
owner record, a journal, placements on disk and whatever the server is
still holding for the connection. The session hosts are not killed.
They are the watcher's children and see their pipe close, which is
what ends them when Console dies; returns their pids so the caller
can see that they did.
"""
sessions = self.session_pids()
doomed = [pid for pid in self.process_tree() if pid not in sessions]
for pid in sorted(doomed, reverse=True):
try:
os.kill(pid, signal.SIGKILL)
except ProcessLookupError:
continue
deadline = time.monotonic() + 15.0
while any(alive(pid) for pid in doomed):
if time.monotonic() >= deadline:
raise RuntimeError(
f"{len([pid for pid in doomed if alive(pid)])} controller "
"process(es) survived SIGKILL, so whatever is started next "
"would be the agent's second controller rather than its only one"
)
time.sleep(0.05)
return sessions
def start_controller(self, bundle: Path) -> None:
"""Start a controller on this root again, after one was stopped.
The state it starts from is whatever the last one left on disk, which
after a kill includes its own owner record: reclaiming that is part of
what a restart has to do. `bundle` is the build to start it from — the
same one to restart the controller that was there, another to put the
machine through the upgrade a user performs by updating the app over
its own state.
"""
superseded = self.supervisor_pid
self.bundle = bundle
_start_watcher(
bundle=bundle,
root=self.root,
template_path=self.template_path,
environment=self.environment,
)
self.supervisor_pid = _await_supervisor(self.root, superseded)
def _template(
*,
agent_id: str,
credentials_path: Path,
cwd: Path,
connection_id: str,
) -> dict[str, object]:
session_id = str(uuid.uuid4())
return {
"session": {
"sessionId": session_id,
"agentId": agent_id,
"provider": "claude",
"hostId": str(uuid.uuid4()),
"epoch": str(uuid.uuid4()),
"status": "starting",
"connectivity": "online",
"capabilities": SESSION_CAPABILITIES,
"pendingRequestIds": [],
},
"start": {
"provider": "claude",
"input": {
"sessionId": session_id,
"cwd": str(cwd),
"runtimeMode": "approval-required",
"env": {},
"mcpServers": {},
},
},
"roomConnection": {"connectionId": connection_id, "rooms": []},
"execution": {
"credentialsPath": str(credentials_path),
"inheritEnv": [],
"codexConfig": "",
"skill": "",
"context": "",
},
}
@contextmanager
def bench_watcher(
*,
bundle: Path,
home: Path,
base_url: str,
agent_id: str,
api_key: str,
environment: Mapping[str, str],
) -> Iterator[BenchWatcher]:
"""Start a watcher for one agent and stop its whole tree afterwards.
It opens the connection Switch Console derives for the agent. `environment`
is added to the one every process in the tree runs with, such as a park
timeout for a scenario about sessions parking.
"""
connection_id = controller_connection_id(agent_id)
root = home / "watch"
root.mkdir(parents=True)
trace_path = home / "host-trace.jsonl"
trace_path.touch()
credentials_path = home / "credentials.json"
credentials_path.write_text(
json.dumps(
{
"env": {
"SWITCH_API_ENDPOINT": base_url,
"SWITCH_API_TOKEN": api_key,
"SWITCH_AGENT_ID": agent_id,
}
}
)
)
# Connected and allowed to start sessions: the workload addresses rooms
# nothing is running for, so a controller that may not spawn would measure
# a topology with no sessions in it.
(root / "watch.json").write_text(json.dumps({"enabled": True, "spawn": True}))
template_path = home / "template.json"
template_path.write_text(
json.dumps(
_template(
agent_id=agent_id,
credentials_path=credentials_path,
cwd=home,
connection_id=connection_id,
)
)
)
environment = {
**os.environ,
**environment,
"HOME": str(home),
"SWITCH_BENCH_TRACE": str(trace_path),
}
_start_watcher(
bundle=bundle, root=root, template_path=template_path, environment=environment
)
watcher = BenchWatcher(
root=root,
home=home,
trace_path=trace_path,
bundle=bundle,
connection_id=connection_id,
template_path=template_path,
environment=environment,
supervisor_pid=_await_supervisor(root, None),
)
try:
yield watcher
finally:
_terminate(watcher.process_tree)
def _start_watcher(
*,
bundle: Path,
root: Path,
template_path: Path,
environment: Mapping[str, str],
) -> None:
"""Ask for a watcher on this root, whether or not one has run here before."""
started = subprocess.run(
["node", str(bundle), str(root), str(template_path), "--ensure-watch"],
capture_output=True,
text=True,
env=dict(environment),
timeout=120,
)
if started.returncode != 0:
raise RuntimeError(
f"the benchmark watcher did not start.\n{started.stdout}\n{started.stderr}"
)
def _await_supervisor(root: Path, superseded: int | None) -> int:
"""The pid of the detached supervisor that now owns this root.
Read from its own owner record rather than from the process that launched
it: that process spawns the supervisor detached and exits, so the
supervisor is reparented and is not in the driver's process tree.
`superseded` is the controller this one replaces, if there was one. A
killed controller leaves its owner record behind, so without it the caller
would be handed the pid of the process it has just killed.
"""
owner = root / "supervisor" / "owner.json"
failure = root / "supervisor" / "failure.json"
deadline = time.monotonic() + 30.0
while time.monotonic() < deadline:
if failure.exists():
raise RuntimeError(
f"the benchmark watcher failed at startup: "
f"{json.loads(failure.read_text())['message']}"
)
if owner.exists():
pid = int(json.loads(owner.read_text())["pid"])
if pid != superseded:
return pid
time.sleep(0.05)
raise RuntimeError(f"no watcher supervisor claimed {root} within 30s")
def _terminate(tree: Callable[[], list[int]]) -> None:
"""Stop a detached host tree, children first, and wait for it to be gone.
The tree is re-read on every pass and accumulated rather than snapshotted
once. A watcher spawns hosts continuously, so a single scan misses any host
started between the scan and the signal; and once the supervisor dies its
children are reparented, so they stop being reachable from it and a later
scan would report the tree empty while they were still running.
A survivor is not untidiness. The workloads share a machine and run one
after another, so a host left serving the previous workload is CPU, memory
and an inbound connection that the next workload's figures would be quietly
charged for.
"""
known: set[int] = set()
for signum, grace in ((signal.SIGTERM, 15.0), (signal.SIGKILL, 10.0)):
deadline = time.monotonic() + grace
while True:
known.update(tree())
# Highest pid first, so a worker is signalled before the supervisor
# that would otherwise notice it die and start a replacement.
living = sorted((pid for pid in known if alive(pid)), reverse=True)
if not living:
return
for pid in living:
try:
os.kill(pid, signum)
except ProcessLookupError:
continue
if time.monotonic() >= deadline:
break
time.sleep(0.1)
remaining = [pid for pid in known if alive(pid)]
if remaining:
raise RuntimeError(
f"{len(remaining)} benchmark host process(es) survived SIGKILL: "
f"{sorted(remaining)}. Anything measured after this would be sharing "
"the machine with a workload that was supposed to be over."
)
def _owner(lock: Path) -> int | None:
"""The live process a lock file names, or None when there is none."""
try:
pid = int(json.loads(lock.read_text())["pid"])
except FileNotFoundError:
return None
return pid if alive(pid) else None
def alive(pid: int) -> bool:
"""Whether a process with this pid exists."""
try:
os.kill(pid, 0)
except ProcessLookupError:
return False
except PermissionError:
return True
return True
def await_dispatches(
watcher: BenchWatcher, markers: Mapping[str, str], timeout: float
) -> frozenset[str]:
"""Block until every marker has been dispatched, or the deadline passes.
Returns the correlations that were never dispatched, rather than raising on
them: a loss is a measurement to report beside the latency of what did
arrive, not something that should throw that latency away.
The caller decides what the loss means. Where every message is expected to
arrive, the caller asserts the result is empty, so nothing is measured over
a silently partial population.
"""
outstanding = set(markers)
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
outstanding -= dispatched(watcher, markers)
if not outstanding:
break
time.sleep(0.05)
return frozenset(markers[marker] for marker in outstanding)
def dispatched(watcher: BenchWatcher, markers: Mapping[str, str]) -> frozenset[str]:
"""Which of `markers` a provider has already been handed, read right now.
Asked rather than waited for, so a scenario that interrupts a delivery can
say whether it interrupted anything.
"""
served = {
record["correlation"] for record in host_records(watcher, PROVIDER_DISPATCH)
}
return frozenset(marker for marker in markers if marker in served)
def host_records(watcher: BenchWatcher, point: str) -> list[dict[str, Any]]:
"""Every record the watcher's hosts traced at `point`, in the order written."""
return [
record
for line in watcher.trace_path.read_text().splitlines()
if line.strip()
for record in (json.loads(line),)
if record["point"] == point
]