Open-source AI teammates in Slack, Teams, Discord, Telegram and Mattermost. Self-host it, with any agent harness and any model.
Python agent-orchestrationaiai-agentschatopsdiscordlangchainmcpmicrosoft-teamsmulti-agentopen-sourceopenaiself-hostedslack
"""Assembles observability from config and the running process's own objects.
Apart from ``main`` so what is measured can be read and tested without starting
a server. The health monitor always runs whatever the config says: readiness is
how Kubernetes routes traffic, and cannot depend on whether anyone is
collecting metrics. Only the export is gated.
"""
from __future__ import annotations
import asyncio
import logging
import socket
from collections.abc import Callable, Iterator, Mapping, Sequence
from dataclasses import dataclass
import httpx
from sqlalchemy.ext.asyncio import AsyncEngine, async_sessionmaker
from switch_core.config import SwitchConfig
from switch_core.logging_context import LogContextFilter
from switch_core.observability.catalogue import (
AGENTS_CONNECTED,
BRIDGES_CRASHED,
BRIDGES_RUNNING,
CONSUMERS_RUNNING,
DB_POOL_IN_USE,
DB_POOL_OVERFLOW,
DB_POOL_SIZE,
DELIVERY_CACHE_BYTES,
DELIVERY_CACHE_ROOMS,
RUNTIME_STARTS,
)
from switch_core.observability.db_server import DbServerSampler
from switch_core.observability.exporter import MetricsExporter
from switch_core.observability.health import (
HealthMonitor,
agent_event_listener_check,
bridges_check,
database_check,
message_listener_check,
session_activity_listener_check,
signal_bus_check,
)
from switch_core.observability.logs import (
DEFAULT_BATCH_SIZE,
DEFAULT_QUEUE_CAPACITY,
LogExporter,
OtlpLogHandler,
)
from switch_core.observability.metrics import (
GaugeReading,
MetricsRegistry,
install,
uninstall,
)
from switch_core.observability.otlp import OtlpClient, OtlpResource
from switch_core.observability.pool import PoolStats
from switch_core.observability.runtime import (
EventLoopLag,
GcPauses,
RuntimeMetrics,
log_unreadable_sources,
watch_event_loop,
)
from switch_core.transport.room_cache import RoomCacheStats
from switch_core.version import switch_core_commit, switch_core_repository_url
logger = logging.getLogger(__name__)
# Matched to the kubelet's probe period; longer and a fault is reported one
# probe later than it could have been.
HEALTH_REFRESH_INTERVAL_SECONDS = 10.0
# Far shorter than the metric interval: a metric interval is a bucket, a log is
# read by someone looking at an incident now.
LOG_EXPORT_INTERVAL_SECONDS = 5.0
@dataclass(frozen=True)
class RuntimeProbes:
"""Live state, as callables over whatever holds it.
Callables rather than the services, so this module does not depend on half
the application and a test can supply the numbers without a bridge.
"""
listener_connected: Callable[[], bool]
session_activity_listener_connected: Callable[[], bool]
agent_event_listener_connected: Callable[[], bool]
signal_bus_connected: Callable[[], bool]
bridges_running: Callable[[], int]
bridges_running_by_platform: Callable[[], Mapping[str, int]]
bridges_configured: Callable[[], int]
# (platform, display name) per bridge whose task has died.
bridges_crashed: Callable[[], Sequence[tuple[str, str]]]
consumers_running: Callable[[], int]
# Agents per (transport, client); see
# `AgentConnectionRegistry.live_agents_by_transport`.
agents_connected: Callable[[], Mapping[tuple[str, str], int]]
# None when the engine's pool does not keep these — see
# :mod:`switch_core.observability.pool`.
pool_stats: Callable[[], PoolStats | None]
room_cache_stats: Callable[[], RoomCacheStats]
@dataclass
class Observability:
"""What the server needs to hold on to: the monitor, and how to stop."""
monitor: HealthMonitor
lag: EventLoopLag
_tasks: list[asyncio.Task[None]]
_http_client: httpx.AsyncClient | None
_log_handler: OtlpLogHandler | None
_gc_pauses: GcPauses | None
async def aclose(self) -> None:
# Detached first, so shutdown logging is not queued for an exporter
# about to stop draining it.
if self._log_handler is not None:
logging.getLogger().removeHandler(self._log_handler)
if self._gc_pauses is not None:
self._gc_pauses.uninstall()
for task in self._tasks:
task.cancel()
for task in self._tasks:
try:
await task
except asyncio.CancelledError:
pass
except Exception:
logger.exception("An observability task failed on shutdown.")
if self._http_client is not None:
await self._http_client.aclose()
uninstall()
def _state_readings(probes: RuntimeProbes) -> Callable[[], Iterator[GaugeReading]]:
# Every (transport, client) pair reported so far: one that empties out is
# reported as 0, or the dashboard would keep showing its last count.
seen: set[tuple[str, str]] = {("websocket", "unknown")}
# Every bridge reported down so far: once it runs again it reads 0, so its
# alert recovers instead of going quiet.
crashed_seen: set[tuple[str, str]] = set()
def readings() -> Iterator[GaugeReading]:
connected = probes.agents_connected()
seen.update(connected)
for transport, client in sorted(seen):
yield GaugeReading(
AGENTS_CONNECTED,
float(connected.get((transport, client), 0)),
{"transport": transport, "client": client},
)
yield GaugeReading(CONSUMERS_RUNNING, float(probes.consumers_running()), {})
for platform, running in probes.bridges_running_by_platform().items():
yield GaugeReading(
BRIDGES_RUNNING,
float(running),
{"bridge": "collaboration", "platform": platform},
)
crashed = set(probes.bridges_crashed())
crashed_seen.update(crashed)
for platform, name in sorted(crashed_seen):
yield GaugeReading(
BRIDGES_CRASHED,
1.0 if (platform, name) in crashed else 0.0,
{"platform": platform, "bridge_name": name},
)
stats = probes.pool_stats()
if stats is not None:
yield GaugeReading(DB_POOL_IN_USE, float(stats.in_use), {})
yield GaugeReading(DB_POOL_SIZE, float(stats.size), {})
yield GaugeReading(DB_POOL_OVERFLOW, float(stats.overflow), {})
cache = probes.room_cache_stats()
yield GaugeReading(DELIVERY_CACHE_BYTES, float(cache.bytes), {})
yield GaugeReading(DELIVERY_CACHE_ROOMS, float(cache.rooms), {})
return readings
def start_observability(
config: SwitchConfig,
version: str | None,
session_factory: async_sessionmaker,
probes: RuntimeProbes,
process_id: str,
db_server_engine: Callable[[], AsyncEngine] | None = None,
) -> Observability:
"""Install the registry, start the loops, and hand back the handle.
Called once, from the server's lifespan. `db_server_engine` builds the
engine the database sampler takes its one connection from; it must not be
the application's pooled engine. `process_id` names this process on
everything it reports (`process_identity.py`).
"""
# With another replica to take the traffic, a process that would answer
# late is better out of service; with one, it is the only one there is.
listeners_gate = config.core_replicas > 1
monitor = HealthMonitor(
checks=[
database_check(session_factory),
message_listener_check(probes.listener_connected, listeners_gate),
session_activity_listener_check(
probes.session_activity_listener_connected, listeners_gate
),
agent_event_listener_check(
probes.agent_event_listener_connected, listeners_gate
),
signal_bus_check(probes.signal_bus_connected, listeners_gate),
bridges_check(probes.bridges_running, probes.bridges_configured),
],
interval_seconds=HEALTH_REFRESH_INTERVAL_SECONDS,
)
lag = EventLoopLag()
tasks: list[asyncio.Task[None]] = [
asyncio.create_task(monitor.run_forever(), name="health-monitor")
]
if not config.observability_enabled:
logger.info(
"No OTLP_ENDPOINT is configured, so nothing is reported off this "
"server. Health checks still run and /health/ready still answers."
)
return Observability(
monitor=monitor,
lag=lag,
_tasks=tasks,
_http_client=None,
_log_handler=None,
_gc_pauses=None,
)
registry = MetricsRegistry()
install(registry)
registry.increment(RUNTIME_STARTS, {})
monitor.install(registry)
gc_pauses = GcPauses()
gc_pauses.install()
RuntimeMetrics(lag, gc_pauses).install(registry)
tasks.append(asyncio.create_task(watch_event_loop(lag), name="event-loop-probe"))
registry.register_observer(_state_readings(probes))
log_unreadable_sources()
http_client = httpx.AsyncClient()
client = OtlpClient(
base_endpoint=str(config.otlp_endpoint),
timeout_seconds=config.otlp_timeout_seconds,
headers=config.otlp_header_map,
client=http_client,
)
resource = OtlpResource(
service_name=config.service_name,
service_version=version,
environment=config.environment,
deployment_id=str(config.deployment_id),
commit_sha=switch_core_commit(),
repository_url=switch_core_repository_url(),
instance_id=process_id,
pod_name=config.pod_name or socket.gethostname(),
)
if config.otlp_metrics_enabled:
exporter = MetricsExporter(
registry=registry,
client=client,
resource=resource,
interval_seconds=config.otlp_export_interval_seconds,
)
tasks.append(
asyncio.create_task(exporter.run_forever(), name="metrics-exporter")
)
if db_server_engine is not None:
sampler = DbServerSampler(db_server_engine)
registry.register_observer(sampler.readings)
tasks.append(
asyncio.create_task(sampler.run_forever(), name="db-server-sampler")
)
logger.info(
"Reporting metrics to %s every %.0fs as service %r.",
client.url_for("metrics"),
config.otlp_export_interval_seconds,
config.service_name,
)
else:
logger.warning(
"OTLP_ENDPOINT is set but OTLP_METRICS_ENABLED is false, so no "
"metrics are being reported."
)
log_handler: OtlpLogHandler | None = None
if config.otlp_logs_enabled:
log_handler = OtlpLogHandler(capacity=DEFAULT_QUEUE_CAPACITY)
# The same filter the stderr handler carries; without it a shipped
# record has no tenant, request or agent on it.
log_handler.addFilter(LogContextFilter(config.tenant_id))
logging.getLogger().addHandler(log_handler)
tasks.append(
asyncio.create_task(
LogExporter(
handler=log_handler,
client=client,
resource=resource,
interval_seconds=LOG_EXPORT_INTERVAL_SECONDS,
batch_size=DEFAULT_BATCH_SIZE,
).run_forever(),
name="log-exporter",
)
)
logger.info(
"Also shipping logs to %s. They continue to be written to this "
"container's output, which remains the primary copy.",
client.url_for("logs"),
)
return Observability(
monitor=monitor,
lag=lag,
_tasks=tasks,
_http_client=http_client,
_log_handler=log_handler,
_gc_pauses=gc_pauses,
)