Skip to content

Open-source AI teammates in Slack, Teams, Discord, Telegram and Mattermost. Self-host it, with any agent harness and any model.

27.5 KB
Raw Blame
"""A real Switch agent-bridge server on a real socket, for benchmarking.

The point of this module is what it does *not* do. It calls
`create_agent_bridge_app` with the same arguments `switch_core.main` passes, and
serves the app it gets back over uvicorn on a loopback port. Nothing is
reimplemented, stubbed or monkeypatched: routing, authentication, the
connection registry and its placements, the event buffer, the SSE stream, the
session activity routes and the approval outcomes are the shipped ones, so a
measurement taken here is a measurement of the server.

Two things are added from the outside, both wrapping objects this module owns
rather than anything under `switch_core`: an ASGI layer around the returned app,
and a commit listener on the engine the fixtures built. See `instrumentation`.

A real socket rather than an in-process transport because the thing under study
is a connection model. `TestClient` and ASGI transports collapse the socket, the
event loop handoff and the keep-alive behaviour that a connection-count question
is entirely about.
"""

from __future__ import annotations

import asyncio
import socket
from collections import Counter
from collections.abc import AsyncIterator
from contextlib import AbstractAsyncContextManager, asynccontextmanager
from dataclasses import dataclass
from typing import cast

import uvicorn
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker

from switch_core.bridges.agent.app import create_agent_bridge_app
from switch_core.bridges.agent.protocol.agent_connections import AgentConnectionRegistry
from switch_core.bridges.agent.protocol.agent_core import AgentCore
from switch_core.bridges.agent.protocol.agent_event_listener import AgentEventListener
from switch_core.bridges.agent.protocol.event_buffer import (
    EventBuffer,
    session_slots_for,
)
from switch_core.bridges.agent.protocol.types import (
    IntegrationProfile,
)
from switch_core.bridges.resource.service import ResourceService
from switch_core.clients.actor import Actor, AgentActor, HumanActor
from switch_core.clients.client_factory import ClientFactory
from switch_core.clients.client_lifecycle_service import ClientLifecycleService
from switch_core.db.engine import create_unpooled_engine
from switch_core.db.models import (
    TENANT_ZERO_ID,
    AgentSessionActivityItem,
    ApprovalRequest,
    Message,
    User,
    agent_event_boot_sequence,
)
from switch_core.db.session_scope import tenant_session
from switch_core.db.stores.budget_store import BudgetStore
from switch_core.db.stores.delivery_cursor_store import DeliveryCursorStore
from switch_core.db.stores.ownership_lease_store import OwnershipLeaseStore
from switch_core.db.stores.switch_core_process_store import SwitchCoreProcessStore
from switch_core.db.stores.tenant_store import TenantStore
from switch_core.db.stores.usage_store import UsageStore
from switch_core.delivery.conversation import ConversationRules, ConversationSettings
from switch_core.delivery.recipient import RecipientServices
from switch_core.delivery.router import MessageRouter
from switch_core.main import (
    _agent_event_retention_loop,
    _connection_sweep_loop,
    _runtime_state_sweep_loop,
    _seed_agent_registration_bootstrap_key,
)
from switch_core.messages.notify import MessageListener
from switch_core.observability.runtime import EventLoopLag
from switch_core.ownership import Ownership
from switch_core.provisioning import Provisioning
from switch_core.provisioning.postgres import PostgresProvisioning
from switch_core.room_service import RoomCreateConfig, RoomService
from switch_core.session_activity.listener import AgentSessionActivityListener
from switch_core.session_activity.outcomes import ApprovalOutcomes
from switch_core.session_activity.service import AgentSessionActivityService, SwitchUser
from switch_core.sessions.contract import ApprovalResult
from switch_core.signals import SignalBus
from switch_core.telemetry.messages import MessageTelemetry
from switch_core.telemetry.service import TelemetryService
from switch_core.telemetry.sink import NullSink
from switch_core.tenant_context import bind_tenant_id, tenant_scope
from switch_core.transport.cursors import CursorWriter
from switch_core.transport.ephemeral import EphemeralBus
from switch_core.transport.invites import InviteBus
from switch_core.transport.room_cache import RoomCacheLimits, RoomDeliveryCache
from switch_core.trust.client import NullTrustClient
from tests.benchmarks.instrumentation import (
    RequestCounter,
    TracingMiddleware,
    count_statements,
    reply_correlation,
    trace_commits,
)
from tests.benchmarks.trace import TraceCollector
from tests.integration.conftest import (
    GATEWAY_ADMIN_EMAIL,
    SessionEnv,
    _NoBridges,
    _truncate_all,
)
from tests.ownership import process_lease_on

# Agents driven by a host process, which is what the benchmark spawns. The
# profile has to say so: `session_passive` would suppress the addressed
# delivery the whole measurement is built on.
BENCH_PROFILE = IntegrationProfile(
    connection_model="always_on",
    message_exchange=True,
    pre_invocation_mediation=[],
    post_invocation_mediation=[],
    event_reporting=[],
)


@dataclass(frozen=True, slots=True)
class BenchAgent:
    agent_id: str
    name: str
    api_key: str


class _Server(uvicorn.Server):
    def install_signal_handlers(self) -> None:
        """Leave the test runner's signal handling alone.

        uvicorn replaces SIGINT and SIGTERM on `serve()`. In-process that would
        take Ctrl-C away from pytest for the rest of the run.
        """
        return


class BenchServer:
    """A running agent bridge, plus the few setup operations a run needs."""

    def __init__(
        self,
        *,
        base_url: str,
        port: int,
        protocol: AgentCore,
        room_service: RoomService,
        client_lifecycle: ClientLifecycleService,
        event_buffer: EventBuffer,
        connections: AgentConnectionRegistry,
        collector: TraceCollector,
        requests: RequestCounter,
        statements: Counter[str],
        owner_id: str,
        session_factory: async_sessionmaker[AsyncSession],
        activity: AgentSessionActivityService,
        agents: tuple[BenchAgent, ...],
    ) -> None:
        self.base_url = base_url
        self.port = port
        self.protocol = protocol
        self.room_service = room_service
        self.client_lifecycle = client_lifecycle
        self.event_buffer = event_buffer
        self.connections = connections
        self.collector = collector
        self.requests = requests
        self.statements = statements
        self.owner_id = owner_id
        self._session_factory = session_factory
        self._activity = activity
        self._agents: list[BenchAgent] = list(agents)

    @property
    def agents(self) -> tuple[BenchAgent, ...]:
        """The agents registered here, for a Core that takes this one's place.

        Their keys are held by host processes that outlive a restart, so a
        replacement adopts them rather than registering the agents again.
        """
        return tuple(self._agents)

    def placed_sessions(
        self, agent_id: str, room_ids: list[str]
    ) -> dict[str, str | None]:
        """Per room, the session of this agent Switch has working in it."""
        return {
            room_id: self.connections.session_in_room(agent_id, room_id)
            for room_id in room_ids
        }

    async def replies(self) -> dict[str, list[str]]:
        """Every benchmark reply stored, by the message it answers: the rooms it landed in.

        Read from the messages table rather than from what the session was
        told, so a reply Switch put in the wrong room shows as that room.
        """
        async with tenant_session(self._session_factory, TENANT_ZERO_ID) as db:
            rows = (
                await db.execute(
                    select(Message.room_id, Message.body).where(
                        Message.body.like("switch-bench-reply:%")
                    )
                )
            ).all()
        found: dict[str, list[str]] = {}
        for room_id, body in rows:
            correlation = reply_correlation(body or "")
            if correlation is not None:
                found.setdefault(correlation, []).append(room_id)
        return found

    async def activity_rows(self, agent_id: str) -> list[AgentSessionActivityItem]:
        """The turn-step rows this agent's session hosts reported."""
        async with tenant_session(self._session_factory, TENANT_ZERO_ID) as db:
            return list(
                (
                    await db.execute(
                        select(AgentSessionActivityItem).where(
                            AgentSessionActivityItem.agent_id == agent_id
                        )
                    )
                )
                .scalars()
                .all()
            )

    async def approval_requests(self, agent_id: str) -> list[ApprovalRequest]:
        """The requests this agent's sessions opened, as Switch holds them now."""
        async with tenant_session(self._session_factory, TENANT_ZERO_ID) as db:
            return list(
                (
                    await db.execute(
                        select(ApprovalRequest).where(
                            ApprovalRequest.agent_id == agent_id
                        )
                    )
                )
                .scalars()
                .all()
            )

    async def answer_approval(self, request: ApprovalRequest, decision: str) -> None:
        """Answer a request as the agent's owner, through the shipped service.

        The option is chosen by the decision it carries, which is what a
        person pressing a button on a card chooses.
        """
        option = next(
            option for option in request.options if option["decision"] == decision
        )
        await self._activity.answer_approval(
            request.agent_id,
            request.session_id,
            request.request_id,
            answer=ApprovalResult(kind="approval", option_id=option["id"]),
            answerer=SwitchUser(self.owner_id),
        )

    async def register_agent(self, name: str) -> BenchAgent:
        result = await self.protocol.register_agent(
            name=name,
            description=f"benchmark agent {name}",
            connector_type="claude-code",
            integration_profile=BENCH_PROFILE,
            owner_id=self.owner_id,
            overwrite=True,
            addressing_policy=None,
        )
        agent = BenchAgent(agent_id=result.agent_id, name=name, api_key=result.api_key)
        self._agents.append(agent)
        return agent

    async def start_clients(self, timeout: float) -> None:
        await self.client_lifecycle.start_all()
        deadline = asyncio.get_event_loop().time() + timeout
        for agent in self._agents:
            client = await self._await_client(agent.agent_id, deadline)
            await client.wait_ready()

    async def _await_client(self, agent_id: str, deadline: float) -> AgentActor:
        while asyncio.get_event_loop().time() < deadline:
            client = self.client_lifecycle.get_by_agent_id(agent_id)
            if client is not None:
                return client  # type: ignore[return-value]
            await asyncio.sleep(0.05)
        raise RuntimeError(f"agent {agent_id} never started a client")

    async def create_room(self, name: str, agent_ids: list[str]) -> str:
        result = await self.room_service.create_room(
            RoomCreateConfig(
                name=name,
                description="connection-model baseline benchmark room",
                agent_ids=agent_ids,
            )
        )
        return result.room.id

    async def address(
        self, *, sender: BenchAgent, room_id: str, target: str, body: str
    ) -> str:
        """Post a message addressed to `target`, returning its message id.

        The message id is half the correlation every trace point is keyed on,
        so a caller that drops it cannot score the event it just created.
        """
        result = await self.protocol.send_targeted_message(
            sender.agent_id, room_id, [target], body
        )
        return result.event_id


@asynccontextmanager
async def bench_server(
    session_env: SessionEnv, collector: TraceCollector
) -> AsyncIterator[BenchServer]:
    """Boot the agent bridge on a loopback port for the life of the block."""
    owner_id = await _prepare(session_env)
    async with _serve(
        session_env=session_env,
        collector=collector,
        owner_id=owner_id,
        port=0,
        agents=(),
    ) as server:
        yield server


class BenchCore:
    """A bench server that can be stopped and started again on its own database.

    Restarting is the whole point of it. The agent bridge holds the replay
    buffer, the connection registry and the session placements in memory, so a
    Core that comes back has lost all three while every row it wrote is still
    there; what the watcher holds is what it has to come back to.

    The port is kept across the restart. Hosts that outlived the Core were
    pointed at an address and go on retrying it, as they would against a
    restarted server at a fixed address; moving the port would make the
    scenario measure reconfiguration instead of recovery.
    """

    def __init__(
        self,
        *,
        session_env: SessionEnv,
        collector: TraceCollector,
        owner_id: str,
        server: BenchServer,
        running: AbstractAsyncContextManager[BenchServer],
    ) -> None:
        self._session_env = session_env
        self._collector = collector
        self._owner_id = owner_id
        self._server = server
        self._running = running

    @property
    def server(self) -> BenchServer:
        """The Core that is serving now, which a restart replaces."""
        return self._server

    async def restart(self) -> None:
        port, agents = self._server.port, self._server.agents
        await self._running.__aexit__(None, None, None)
        self._running = _serve(
            session_env=self._session_env,
            collector=self._collector,
            owner_id=self._owner_id,
            port=port,
            agents=agents,
        )
        self._server = await self._running.__aenter__()

    async def aclose(self) -> None:
        await self._running.__aexit__(None, None, None)


@asynccontextmanager
async def restartable_bench_server(
    session_env: SessionEnv, collector: TraceCollector
) -> AsyncIterator[BenchCore]:
    """The same server, wrapped so a scenario can restart it mid-run."""
    owner_id = await _prepare(session_env)
    running = _serve(
        session_env=session_env,
        collector=collector,
        owner_id=owner_id,
        port=0,
        agents=(),
    )
    core = BenchCore(
        session_env=session_env,
        collector=collector,
        owner_id=owner_id,
        server=await running.__aenter__(),
        running=running,
    )
    try:
        yield core
    finally:
        await core.aclose()


async def _prepare(session_env: SessionEnv) -> str:
    """Empty the database and seed what a Core needs before it serves.

    Separate from serving because a Core that replaces another one must not do
    it: the agents, rooms and sessions in that database are exactly what the
    restart is supposed to come back to.
    """
    await _truncate_all(session_env.owner_engine)
    session_factory = cast(
        "async_sessionmaker[AsyncSession]", session_env.session_factory
    )
    with tenant_scope(TENANT_ZERO_ID):
        owner = User(name="Admin", email=GATEWAY_ADMIN_EMAIL, role="admin")
        async with session_factory() as session:
            await session_env.user_store.create(session, owner)
            await session.commit()
        await _seed_agent_registration_bootstrap_key(
            session_factory,
            session_env.user_store,
            session_env.api_key_store,
            session_env.agent_store,
            session_env.config,
        )
    return owner.id


@asynccontextmanager
async def _serve(
    *,
    session_env: SessionEnv,
    collector: TraceCollector,
    owner_id: str,
    port: int,
    agents: tuple[BenchAgent, ...],
) -> AsyncIterator[BenchServer]:
    """Serve the agent bridge until the block ends.

    Only one may run per process: `init_dependencies` stores the wiring in a
    module-level dict that the HTTP routes resolve against, so a second server
    would silently repoint the first one's routes at its own stores. One after
    another is what a restart is, and is fine.

    `port` is 0 for a free port, or the port a previous Core was serving on.
    `agents` are the ones already registered in the database, whose keys the
    hosts still hold.
    """
    config = session_env.config
    # The fixture types this as `object`; every consumer below needs the real
    # signature, so narrow it once here rather than at each call.
    session_factory = cast(
        "async_sessionmaker[AsyncSession]", session_env.session_factory
    )

    # Bound rather than scoped, and deliberately never unbound: a benchmark
    # process serves one tenant, and a Core that replaces another is entered
    # and left from different asyncio contexts, where resetting the token
    # raises instead of unbinding.
    bind_tenant_id(TENANT_ZERO_ID)
    # Numbered from this boot's base as `main` numbers it, so a Core that
    # replaces another starts above every number the last one handed out and
    # a host's cursor from before the restart reads as older, as it would in
    # production. The events themselves are rows and outlive the restart.
    async with session_env.engine.begin() as connection:
        event_boot = await connection.scalar(agent_event_boot_sequence.next_value())
    message_listener = MessageListener(lambda: create_unpooled_engine(config))
    await message_listener.start()
    event_buffer = EventBuffer(
        session_factory,
        sequence_base=event_boot << 32,
        rooms=message_listener,
        retention_seconds=config.agent_event_retention_seconds,
        session_slots=session_slots_for(config.db_pool_size, config.db_max_overflow),
    )
    agent_event_listener = AgentEventListener(
        lambda: create_unpooled_engine(config), event_buffer
    )
    connections = AgentConnectionRegistry()
    collab_lifecycle = _NoBridges()

    await agent_event_listener.start()
    activity_listener = AgentSessionActivityListener(
        lambda: create_unpooled_engine(config)
    )
    await activity_listener.start()
    activity = AgentSessionActivityService(session_factory)
    invites = InviteBus()
    ephemeral = EphemeralBus()

    resource_service = ResourceService(
        reference_store=session_env.reference_store,
        reference_type_store=session_env.reference_type_store,
        document_store=session_env.document_store,
        package_store=session_env.package_store,
        room_link_store=session_env.room_link_store,
        session_factory=session_factory,
    )

    # Telemetry off: these flows are not about reporting, but the
    # message hook must still be wired for the clients to build.
    message_telemetry = MessageTelemetry(
        telemetry=TelemetryService(
            sink=NullSink(),
            enabled=False,
            client_id="",
            version=None,
            telemetry_environment="dev",
            telemetry_internal=True,
            deployment_kind="self_hosted",
        ),
        session_factory=session_factory,
    )

    router = MessageRouter(
        services=RecipientServices(
            session_factory=session_factory,
            event_buffer=event_buffer,
            agent_store=session_env.agent_store,
            room_store=session_env.room_store,
            document_store=session_env.document_store,
            reference_store=session_env.reference_store,
            package_store=session_env.package_store,
            agent_session_store=session_env.agent_session_store,
            room_role_store=session_env.room_role_store,
            external_user_store=session_env.external_user_store,
            connections=connections,
            frontend_base_url=config.dashboard_base_url,
            message_telemetry=message_telemetry,
        ),
        message_store=session_env.message_store,
        client_store=session_env.client_store,
        budget_store=BudgetStore(),
        conversation=ConversationRules(
            settings=ConversationSettings.from_config(config),
            message_store=session_env.message_store,
            room_store=session_env.room_store,
            agent_store=session_env.agent_store,
            room_role_store=session_env.room_role_store,
        ),
        decisions=None,
        decision_max_age_seconds=120,
    )

    provisioning: Provisioning = PostgresProvisioning(
        session_factory=session_factory,
        room_store=session_env.room_store,
        client_store=session_env.client_store,
        router=router,
        invites=invites,
    )

    room_cache = RoomDeliveryCache(
        session_factory=session_factory,
        message_store=session_env.message_store,
        limits=RoomCacheLimits(
            max_bytes=config.room_delivery_cache_max_bytes,
            max_rooms=config.room_delivery_cache_max_rooms,
            max_rows_per_room=config.room_delivery_cache_max_rows_per_room,
            max_age_seconds=config.room_delivery_cache_max_age_seconds,
        ),
    )

    client_factory = ClientFactory(
        client_store=session_env.client_store,
        session_factory=session_factory,
        config=config,
        room_store=session_env.room_store,
        message_store=session_env.message_store,
        usage_store=UsageStore(),
        media_store=session_env.media_store,
        listener=message_listener,
        invites=invites,
        ephemeral=ephemeral,
        room_cache=room_cache,
        message_observer=message_telemetry,
        router=router,
    )
    client_factory.register("agent", AgentActor)
    client_factory.register("user", HumanActor)
    client_factory.register("bridge", Actor)

    # This process's lease and what it owns, as `main` wires them. Only agent
    # clients run here, which have no consumer, so nothing is ever claimed;
    # the signal bus is never started.
    lease = process_lease_on(session_factory)
    client_lifecycle = ClientLifecycleService(
        provisioning=provisioning,
        client_store=session_env.client_store,
        tenant_store=TenantStore(),
        client_factory=client_factory,
        session_factory=session_factory,
        config=config,
        tenants_isolated=True,
        ownership=Ownership(
            lease=lease,
            session_factory=session_factory,
            bus=SignalBus(
                session_factory,
                lambda: create_unpooled_engine(config),
                process_id=lease.process_id,
            ),
            store=OwnershipLeaseStore(),
            processes=SwitchCoreProcessStore(),
        ),
        cursor_writer=CursorWriter(
            session_factory=session_factory,
            cursors=DeliveryCursorStore(),
            leases=OwnershipLeaseStore(),
            process_id=lease.process_id,
        ),
    )

    room_service = RoomService(
        provisioning=provisioning,
        room_store=session_env.room_store,
        agent_store=session_env.agent_store,
        client_lifecycle=client_lifecycle,
        collab_lifecycle=collab_lifecycle,  # type: ignore[arg-type]
        collab_bridge_store=session_env.bridge_store,
        resource_service=resource_service,
        session_factory=session_factory,
        room_cache=room_cache,
    )

    app, protocol = create_agent_bridge_app(
        agent_store=session_env.agent_store,
        agent_session_store=session_env.agent_session_store,
        room_store=session_env.room_store,
        room_service=room_service,
        client_lifecycle=client_lifecycle,
        collab_lifecycle=collab_lifecycle,  # type: ignore[arg-type]
        event_buffer=event_buffer,
        resource_service=resource_service,
        api_key_store=session_env.api_key_store,
        external_user_store=session_env.external_user_store,
        bridge_store=session_env.bridge_store,
        session_factory=session_factory,
        config=config,
        approval_outcomes=ApprovalOutcomes(activity_listener, activity),
        controller_auth=None,
        trust_client=NullTrustClient(),
        gateway_ui=None,
        connections=connections,
    )

    detach_commits = trace_commits(session_env.engine, collector)
    statements, detach_statements = count_statements(session_env.engine)

    sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
    sock.bind(("127.0.0.1", port))
    sock.listen(2048)
    served_port = sock.getsockname()[1]

    requests = RequestCounter(TracingMiddleware(app, collector))

    server = _Server(
        uvicorn.Config(
            requests,
            log_level="warning",
            access_log=False,
            lifespan="on",
            # The agent protocol's own liveness is the heartbeat; a
            # keep-alive timeout shorter than an idle stream's keepalive
            # interval would close streams the server considers healthy and
            # show up as reconnect churn that the code under study did not
            # cause.
            timeout_keep_alive=120,
        )
    )
    serving = asyncio.create_task(server.serve(sockets=[sock]))
    await _await_started(server, serving)

    # The background sweeps live in `switch_core.main`'s lifespan rather
    # than in the app factory, so a server assembled from the factory alone
    # never reaps a connection whose client stopped beating. The real
    # functions are started here, not copies of them: without the
    # connection sweep a host killed mid-run keeps its stream slot for
    # ever, the server goes on routing that room to a process that is gone,
    # and both the connection count and the recovery case measure a
    # harness defect instead of the topology.
    sweeps = [
        asyncio.create_task(_connection_sweep_loop(protocol, EventLoopLag())),
        asyncio.create_task(_runtime_state_sweep_loop(protocol)),
        asyncio.create_task(_agent_event_retention_loop(session_factory, event_buffer)),
    ]

    bench = BenchServer(
        base_url=f"http://127.0.0.1:{served_port}",
        port=served_port,
        protocol=protocol,
        room_service=room_service,
        client_lifecycle=client_lifecycle,
        event_buffer=event_buffer,
        connections=connections,
        collector=collector,
        requests=requests,
        statements=statements,
        owner_id=owner_id,
        session_factory=session_factory,
        activity=activity,
        agents=agents,
    )
    try:
        yield bench
    finally:
        for sweep in sweeps:
            sweep.cancel()
        await asyncio.gather(*sweeps, return_exceptions=True)
        server.should_exit = True
        await serving
        detach_commits()
        detach_statements()
        await client_lifecycle.stop_all()
        await activity_listener.stop()
        await agent_event_listener.stop()
        await message_listener.stop()
        await provisioning.close()


async def _await_started(server: _Server, serving: asyncio.Task[None]) -> None:
    deadline = asyncio.get_event_loop().time() + 30.0
    while not server.started:
        if serving.done():
            # Surfaces the startup exception rather than timing out on it.
            serving.result()
            raise RuntimeError("the benchmark server exited before it started")
        if asyncio.get_event_loop().time() > deadline:
            raise RuntimeError("the benchmark server did not start within 30s")
        await asyncio.sleep(0.01)