Open-source AI teammates in Slack, Teams, Discord, Telegram and Mattermost. Self-host it, with any agent harness and any model.
Python agent-orchestrationaiai-agentschatopsdiscordlangchainmcpmicrosoft-teamsmulti-agentopen-sourceopenaiself-hostedslack
"""A real Switch agent-bridge server on a real socket, for benchmarking.
The point of this module is what it does *not* do. It calls
`create_agent_bridge_app` with the same arguments `switch_core.main` passes, and
serves the app it gets back over uvicorn on a loopback port. Nothing is
reimplemented, stubbed or monkeypatched: routing, authentication, the
connection registry and its placements, the event buffer, the SSE stream, the
session activity routes and the approval outcomes are the shipped ones, so a
measurement taken here is a measurement of the server.
Two things are added from the outside, both wrapping objects this module owns
rather than anything under `switch_core`: an ASGI layer around the returned app,
and a commit listener on the engine the fixtures built. See `instrumentation`.
A real socket rather than an in-process transport because the thing under study
is a connection model. `TestClient` and ASGI transports collapse the socket, the
event loop handoff and the keep-alive behaviour that a connection-count question
is entirely about.
"""
from __future__ import annotations
import asyncio
import socket
from collections import Counter
from collections.abc import AsyncIterator
from contextlib import AbstractAsyncContextManager, asynccontextmanager
from dataclasses import dataclass
from typing import cast
import uvicorn
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from switch_core.bridges.agent.app import create_agent_bridge_app
from switch_core.bridges.agent.protocol.agent_connections import AgentConnectionRegistry
from switch_core.bridges.agent.protocol.agent_core import AgentCore
from switch_core.bridges.agent.protocol.agent_event_listener import AgentEventListener
from switch_core.bridges.agent.protocol.event_buffer import (
EventBuffer,
session_slots_for,
)
from switch_core.bridges.agent.protocol.types import (
IntegrationProfile,
)
from switch_core.bridges.resource.service import ResourceService
from switch_core.clients.actor import Actor, AgentActor, HumanActor
from switch_core.clients.client_factory import ClientFactory
from switch_core.clients.client_lifecycle_service import ClientLifecycleService
from switch_core.db.engine import create_unpooled_engine
from switch_core.db.models import (
TENANT_ZERO_ID,
AgentSessionActivityItem,
ApprovalRequest,
Message,
User,
agent_event_boot_sequence,
)
from switch_core.db.session_scope import tenant_session
from switch_core.db.stores.budget_store import BudgetStore
from switch_core.db.stores.delivery_cursor_store import DeliveryCursorStore
from switch_core.db.stores.ownership_lease_store import OwnershipLeaseStore
from switch_core.db.stores.switch_core_process_store import SwitchCoreProcessStore
from switch_core.db.stores.tenant_store import TenantStore
from switch_core.db.stores.usage_store import UsageStore
from switch_core.delivery.conversation import ConversationRules, ConversationSettings
from switch_core.delivery.recipient import RecipientServices
from switch_core.delivery.router import MessageRouter
from switch_core.main import (
_agent_event_retention_loop,
_connection_sweep_loop,
_runtime_state_sweep_loop,
_seed_agent_registration_bootstrap_key,
)
from switch_core.messages.notify import MessageListener
from switch_core.observability.runtime import EventLoopLag
from switch_core.ownership import Ownership
from switch_core.provisioning import Provisioning
from switch_core.provisioning.postgres import PostgresProvisioning
from switch_core.room_service import RoomCreateConfig, RoomService
from switch_core.session_activity.listener import AgentSessionActivityListener
from switch_core.session_activity.outcomes import ApprovalOutcomes
from switch_core.session_activity.service import AgentSessionActivityService, SwitchUser
from switch_core.sessions.contract import ApprovalResult
from switch_core.signals import SignalBus
from switch_core.telemetry.messages import MessageTelemetry
from switch_core.telemetry.service import TelemetryService
from switch_core.telemetry.sink import NullSink
from switch_core.tenant_context import bind_tenant_id, tenant_scope
from switch_core.transport.cursors import CursorWriter
from switch_core.transport.ephemeral import EphemeralBus
from switch_core.transport.invites import InviteBus
from switch_core.transport.room_cache import RoomCacheLimits, RoomDeliveryCache
from switch_core.trust.client import NullTrustClient
from tests.benchmarks.instrumentation import (
RequestCounter,
TracingMiddleware,
count_statements,
reply_correlation,
trace_commits,
)
from tests.benchmarks.trace import TraceCollector
from tests.integration.conftest import (
GATEWAY_ADMIN_EMAIL,
SessionEnv,
_NoBridges,
_truncate_all,
)
from tests.ownership import process_lease_on
# Agents driven by a host process, which is what the benchmark spawns. The
# profile has to say so: `session_passive` would suppress the addressed
# delivery the whole measurement is built on.
BENCH_PROFILE = IntegrationProfile(
connection_model="always_on",
message_exchange=True,
pre_invocation_mediation=[],
post_invocation_mediation=[],
event_reporting=[],
)
@dataclass(frozen=True, slots=True)
class BenchAgent:
agent_id: str
name: str
api_key: str
class _Server(uvicorn.Server):
def install_signal_handlers(self) -> None:
"""Leave the test runner's signal handling alone.
uvicorn replaces SIGINT and SIGTERM on `serve()`. In-process that would
take Ctrl-C away from pytest for the rest of the run.
"""
return
class BenchServer:
"""A running agent bridge, plus the few setup operations a run needs."""
def __init__(
self,
*,
base_url: str,
port: int,
protocol: AgentCore,
room_service: RoomService,
client_lifecycle: ClientLifecycleService,
event_buffer: EventBuffer,
connections: AgentConnectionRegistry,
collector: TraceCollector,
requests: RequestCounter,
statements: Counter[str],
owner_id: str,
session_factory: async_sessionmaker[AsyncSession],
activity: AgentSessionActivityService,
agents: tuple[BenchAgent, ...],
) -> None:
self.base_url = base_url
self.port = port
self.protocol = protocol
self.room_service = room_service
self.client_lifecycle = client_lifecycle
self.event_buffer = event_buffer
self.connections = connections
self.collector = collector
self.requests = requests
self.statements = statements
self.owner_id = owner_id
self._session_factory = session_factory
self._activity = activity
self._agents: list[BenchAgent] = list(agents)
@property
def agents(self) -> tuple[BenchAgent, ...]:
"""The agents registered here, for a Core that takes this one's place.
Their keys are held by host processes that outlive a restart, so a
replacement adopts them rather than registering the agents again.
"""
return tuple(self._agents)
def placed_sessions(
self, agent_id: str, room_ids: list[str]
) -> dict[str, str | None]:
"""Per room, the session of this agent Switch has working in it."""
return {
room_id: self.connections.session_in_room(agent_id, room_id)
for room_id in room_ids
}
async def replies(self) -> dict[str, list[str]]:
"""Every benchmark reply stored, by the message it answers: the rooms it landed in.
Read from the messages table rather than from what the session was
told, so a reply Switch put in the wrong room shows as that room.
"""
async with tenant_session(self._session_factory, TENANT_ZERO_ID) as db:
rows = (
await db.execute(
select(Message.room_id, Message.body).where(
Message.body.like("switch-bench-reply:%")
)
)
).all()
found: dict[str, list[str]] = {}
for room_id, body in rows:
correlation = reply_correlation(body or "")
if correlation is not None:
found.setdefault(correlation, []).append(room_id)
return found
async def activity_rows(self, agent_id: str) -> list[AgentSessionActivityItem]:
"""The turn-step rows this agent's session hosts reported."""
async with tenant_session(self._session_factory, TENANT_ZERO_ID) as db:
return list(
(
await db.execute(
select(AgentSessionActivityItem).where(
AgentSessionActivityItem.agent_id == agent_id
)
)
)
.scalars()
.all()
)
async def approval_requests(self, agent_id: str) -> list[ApprovalRequest]:
"""The requests this agent's sessions opened, as Switch holds them now."""
async with tenant_session(self._session_factory, TENANT_ZERO_ID) as db:
return list(
(
await db.execute(
select(ApprovalRequest).where(
ApprovalRequest.agent_id == agent_id
)
)
)
.scalars()
.all()
)
async def answer_approval(self, request: ApprovalRequest, decision: str) -> None:
"""Answer a request as the agent's owner, through the shipped service.
The option is chosen by the decision it carries, which is what a
person pressing a button on a card chooses.
"""
option = next(
option for option in request.options if option["decision"] == decision
)
await self._activity.answer_approval(
request.agent_id,
request.session_id,
request.request_id,
answer=ApprovalResult(kind="approval", option_id=option["id"]),
answerer=SwitchUser(self.owner_id),
)
async def register_agent(self, name: str) -> BenchAgent:
result = await self.protocol.register_agent(
name=name,
description=f"benchmark agent {name}",
connector_type="claude-code",
integration_profile=BENCH_PROFILE,
owner_id=self.owner_id,
overwrite=True,
addressing_policy=None,
)
agent = BenchAgent(agent_id=result.agent_id, name=name, api_key=result.api_key)
self._agents.append(agent)
return agent
async def start_clients(self, timeout: float) -> None:
await self.client_lifecycle.start_all()
deadline = asyncio.get_event_loop().time() + timeout
for agent in self._agents:
client = await self._await_client(agent.agent_id, deadline)
await client.wait_ready()
async def _await_client(self, agent_id: str, deadline: float) -> AgentActor:
while asyncio.get_event_loop().time() < deadline:
client = self.client_lifecycle.get_by_agent_id(agent_id)
if client is not None:
return client # type: ignore[return-value]
await asyncio.sleep(0.05)
raise RuntimeError(f"agent {agent_id} never started a client")
async def create_room(self, name: str, agent_ids: list[str]) -> str:
result = await self.room_service.create_room(
RoomCreateConfig(
name=name,
description="connection-model baseline benchmark room",
agent_ids=agent_ids,
)
)
return result.room.id
async def address(
self, *, sender: BenchAgent, room_id: str, target: str, body: str
) -> str:
"""Post a message addressed to `target`, returning its message id.
The message id is half the correlation every trace point is keyed on,
so a caller that drops it cannot score the event it just created.
"""
result = await self.protocol.send_targeted_message(
sender.agent_id, room_id, [target], body
)
return result.event_id
@asynccontextmanager
async def bench_server(
session_env: SessionEnv, collector: TraceCollector
) -> AsyncIterator[BenchServer]:
"""Boot the agent bridge on a loopback port for the life of the block."""
owner_id = await _prepare(session_env)
async with _serve(
session_env=session_env,
collector=collector,
owner_id=owner_id,
port=0,
agents=(),
) as server:
yield server
class BenchCore:
"""A bench server that can be stopped and started again on its own database.
Restarting is the whole point of it. The agent bridge holds the replay
buffer, the connection registry and the session placements in memory, so a
Core that comes back has lost all three while every row it wrote is still
there; what the watcher holds is what it has to come back to.
The port is kept across the restart. Hosts that outlived the Core were
pointed at an address and go on retrying it, as they would against a
restarted server at a fixed address; moving the port would make the
scenario measure reconfiguration instead of recovery.
"""
def __init__(
self,
*,
session_env: SessionEnv,
collector: TraceCollector,
owner_id: str,
server: BenchServer,
running: AbstractAsyncContextManager[BenchServer],
) -> None:
self._session_env = session_env
self._collector = collector
self._owner_id = owner_id
self._server = server
self._running = running
@property
def server(self) -> BenchServer:
"""The Core that is serving now, which a restart replaces."""
return self._server
async def restart(self) -> None:
port, agents = self._server.port, self._server.agents
await self._running.__aexit__(None, None, None)
self._running = _serve(
session_env=self._session_env,
collector=self._collector,
owner_id=self._owner_id,
port=port,
agents=agents,
)
self._server = await self._running.__aenter__()
async def aclose(self) -> None:
await self._running.__aexit__(None, None, None)
@asynccontextmanager
async def restartable_bench_server(
session_env: SessionEnv, collector: TraceCollector
) -> AsyncIterator[BenchCore]:
"""The same server, wrapped so a scenario can restart it mid-run."""
owner_id = await _prepare(session_env)
running = _serve(
session_env=session_env,
collector=collector,
owner_id=owner_id,
port=0,
agents=(),
)
core = BenchCore(
session_env=session_env,
collector=collector,
owner_id=owner_id,
server=await running.__aenter__(),
running=running,
)
try:
yield core
finally:
await core.aclose()
async def _prepare(session_env: SessionEnv) -> str:
"""Empty the database and seed what a Core needs before it serves.
Separate from serving because a Core that replaces another one must not do
it: the agents, rooms and sessions in that database are exactly what the
restart is supposed to come back to.
"""
await _truncate_all(session_env.owner_engine)
session_factory = cast(
"async_sessionmaker[AsyncSession]", session_env.session_factory
)
with tenant_scope(TENANT_ZERO_ID):
owner = User(name="Admin", email=GATEWAY_ADMIN_EMAIL, role="admin")
async with session_factory() as session:
await session_env.user_store.create(session, owner)
await session.commit()
await _seed_agent_registration_bootstrap_key(
session_factory,
session_env.user_store,
session_env.api_key_store,
session_env.agent_store,
session_env.config,
)
return owner.id
@asynccontextmanager
async def _serve(
*,
session_env: SessionEnv,
collector: TraceCollector,
owner_id: str,
port: int,
agents: tuple[BenchAgent, ...],
) -> AsyncIterator[BenchServer]:
"""Serve the agent bridge until the block ends.
Only one may run per process: `init_dependencies` stores the wiring in a
module-level dict that the HTTP routes resolve against, so a second server
would silently repoint the first one's routes at its own stores. One after
another is what a restart is, and is fine.
`port` is 0 for a free port, or the port a previous Core was serving on.
`agents` are the ones already registered in the database, whose keys the
hosts still hold.
"""
config = session_env.config
# The fixture types this as `object`; every consumer below needs the real
# signature, so narrow it once here rather than at each call.
session_factory = cast(
"async_sessionmaker[AsyncSession]", session_env.session_factory
)
# Bound rather than scoped, and deliberately never unbound: a benchmark
# process serves one tenant, and a Core that replaces another is entered
# and left from different asyncio contexts, where resetting the token
# raises instead of unbinding.
bind_tenant_id(TENANT_ZERO_ID)
# Numbered from this boot's base as `main` numbers it, so a Core that
# replaces another starts above every number the last one handed out and
# a host's cursor from before the restart reads as older, as it would in
# production. The events themselves are rows and outlive the restart.
async with session_env.engine.begin() as connection:
event_boot = await connection.scalar(agent_event_boot_sequence.next_value())
message_listener = MessageListener(lambda: create_unpooled_engine(config))
await message_listener.start()
event_buffer = EventBuffer(
session_factory,
sequence_base=event_boot << 32,
rooms=message_listener,
retention_seconds=config.agent_event_retention_seconds,
session_slots=session_slots_for(config.db_pool_size, config.db_max_overflow),
)
agent_event_listener = AgentEventListener(
lambda: create_unpooled_engine(config), event_buffer
)
connections = AgentConnectionRegistry()
collab_lifecycle = _NoBridges()
await agent_event_listener.start()
activity_listener = AgentSessionActivityListener(
lambda: create_unpooled_engine(config)
)
await activity_listener.start()
activity = AgentSessionActivityService(session_factory)
invites = InviteBus()
ephemeral = EphemeralBus()
resource_service = ResourceService(
reference_store=session_env.reference_store,
reference_type_store=session_env.reference_type_store,
document_store=session_env.document_store,
package_store=session_env.package_store,
room_link_store=session_env.room_link_store,
session_factory=session_factory,
)
# Telemetry off: these flows are not about reporting, but the
# message hook must still be wired for the clients to build.
message_telemetry = MessageTelemetry(
telemetry=TelemetryService(
sink=NullSink(),
enabled=False,
client_id="",
version=None,
telemetry_environment="dev",
telemetry_internal=True,
deployment_kind="self_hosted",
),
session_factory=session_factory,
)
router = MessageRouter(
services=RecipientServices(
session_factory=session_factory,
event_buffer=event_buffer,
agent_store=session_env.agent_store,
room_store=session_env.room_store,
document_store=session_env.document_store,
reference_store=session_env.reference_store,
package_store=session_env.package_store,
agent_session_store=session_env.agent_session_store,
room_role_store=session_env.room_role_store,
external_user_store=session_env.external_user_store,
connections=connections,
frontend_base_url=config.dashboard_base_url,
message_telemetry=message_telemetry,
),
message_store=session_env.message_store,
client_store=session_env.client_store,
budget_store=BudgetStore(),
conversation=ConversationRules(
settings=ConversationSettings.from_config(config),
message_store=session_env.message_store,
room_store=session_env.room_store,
agent_store=session_env.agent_store,
room_role_store=session_env.room_role_store,
),
decisions=None,
decision_max_age_seconds=120,
)
provisioning: Provisioning = PostgresProvisioning(
session_factory=session_factory,
room_store=session_env.room_store,
client_store=session_env.client_store,
router=router,
invites=invites,
)
room_cache = RoomDeliveryCache(
session_factory=session_factory,
message_store=session_env.message_store,
limits=RoomCacheLimits(
max_bytes=config.room_delivery_cache_max_bytes,
max_rooms=config.room_delivery_cache_max_rooms,
max_rows_per_room=config.room_delivery_cache_max_rows_per_room,
max_age_seconds=config.room_delivery_cache_max_age_seconds,
),
)
client_factory = ClientFactory(
client_store=session_env.client_store,
session_factory=session_factory,
config=config,
room_store=session_env.room_store,
message_store=session_env.message_store,
usage_store=UsageStore(),
media_store=session_env.media_store,
listener=message_listener,
invites=invites,
ephemeral=ephemeral,
room_cache=room_cache,
message_observer=message_telemetry,
router=router,
)
client_factory.register("agent", AgentActor)
client_factory.register("user", HumanActor)
client_factory.register("bridge", Actor)
# This process's lease and what it owns, as `main` wires them. Only agent
# clients run here, which have no consumer, so nothing is ever claimed;
# the signal bus is never started.
lease = process_lease_on(session_factory)
client_lifecycle = ClientLifecycleService(
provisioning=provisioning,
client_store=session_env.client_store,
tenant_store=TenantStore(),
client_factory=client_factory,
session_factory=session_factory,
config=config,
tenants_isolated=True,
ownership=Ownership(
lease=lease,
session_factory=session_factory,
bus=SignalBus(
session_factory,
lambda: create_unpooled_engine(config),
process_id=lease.process_id,
),
store=OwnershipLeaseStore(),
processes=SwitchCoreProcessStore(),
),
cursor_writer=CursorWriter(
session_factory=session_factory,
cursors=DeliveryCursorStore(),
leases=OwnershipLeaseStore(),
process_id=lease.process_id,
),
)
room_service = RoomService(
provisioning=provisioning,
room_store=session_env.room_store,
agent_store=session_env.agent_store,
client_lifecycle=client_lifecycle,
collab_lifecycle=collab_lifecycle, # type: ignore[arg-type]
collab_bridge_store=session_env.bridge_store,
resource_service=resource_service,
session_factory=session_factory,
room_cache=room_cache,
)
app, protocol = create_agent_bridge_app(
agent_store=session_env.agent_store,
agent_session_store=session_env.agent_session_store,
room_store=session_env.room_store,
room_service=room_service,
client_lifecycle=client_lifecycle,
collab_lifecycle=collab_lifecycle, # type: ignore[arg-type]
event_buffer=event_buffer,
resource_service=resource_service,
api_key_store=session_env.api_key_store,
external_user_store=session_env.external_user_store,
bridge_store=session_env.bridge_store,
session_factory=session_factory,
config=config,
approval_outcomes=ApprovalOutcomes(activity_listener, activity),
controller_auth=None,
trust_client=NullTrustClient(),
gateway_ui=None,
connections=connections,
)
detach_commits = trace_commits(session_env.engine, collector)
statements, detach_statements = count_statements(session_env.engine)
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
sock.bind(("127.0.0.1", port))
sock.listen(2048)
served_port = sock.getsockname()[1]
requests = RequestCounter(TracingMiddleware(app, collector))
server = _Server(
uvicorn.Config(
requests,
log_level="warning",
access_log=False,
lifespan="on",
# The agent protocol's own liveness is the heartbeat; a
# keep-alive timeout shorter than an idle stream's keepalive
# interval would close streams the server considers healthy and
# show up as reconnect churn that the code under study did not
# cause.
timeout_keep_alive=120,
)
)
serving = asyncio.create_task(server.serve(sockets=[sock]))
await _await_started(server, serving)
# The background sweeps live in `switch_core.main`'s lifespan rather
# than in the app factory, so a server assembled from the factory alone
# never reaps a connection whose client stopped beating. The real
# functions are started here, not copies of them: without the
# connection sweep a host killed mid-run keeps its stream slot for
# ever, the server goes on routing that room to a process that is gone,
# and both the connection count and the recovery case measure a
# harness defect instead of the topology.
sweeps = [
asyncio.create_task(_connection_sweep_loop(protocol, EventLoopLag())),
asyncio.create_task(_runtime_state_sweep_loop(protocol)),
asyncio.create_task(_agent_event_retention_loop(session_factory, event_buffer)),
]
bench = BenchServer(
base_url=f"http://127.0.0.1:{served_port}",
port=served_port,
protocol=protocol,
room_service=room_service,
client_lifecycle=client_lifecycle,
event_buffer=event_buffer,
connections=connections,
collector=collector,
requests=requests,
statements=statements,
owner_id=owner_id,
session_factory=session_factory,
activity=activity,
agents=agents,
)
try:
yield bench
finally:
for sweep in sweeps:
sweep.cancel()
await asyncio.gather(*sweeps, return_exceptions=True)
server.should_exit = True
await serving
detach_commits()
detach_statements()
await client_lifecycle.stop_all()
await activity_listener.stop()
await agent_event_listener.stop()
await message_listener.stop()
await provisioning.close()
async def _await_started(server: _Server, serving: asyncio.Task[None]) -> None:
deadline = asyncio.get_event_loop().time() + 30.0
while not server.started:
if serving.done():
# Surfaces the startup exception rather than timing out on it.
serving.result()
raise RuntimeError("the benchmark server exited before it started")
if asyncio.get_event_loop().time() > deadline:
raise RuntimeError("the benchmark server did not start within 30s")
await asyncio.sleep(0.01)