Skip to content

Open-source AI teammates in Slack, Teams, Discord, Telegram and Mattermost. Self-host it, with any agent harness and any model.

11.6 KB
Raw Blame
"""Assembles observability from config and the running process's own objects.

Apart from ``main`` so what is measured can be read and tested without starting
a server. The health monitor always runs whatever the config says: readiness is
how Kubernetes routes traffic, and cannot depend on whether anyone is
collecting metrics. Only the export is gated.
"""

from __future__ import annotations

import asyncio
import logging
import socket
from collections.abc import Callable, Iterator, Mapping, Sequence
from dataclasses import dataclass

import httpx
from sqlalchemy.ext.asyncio import AsyncEngine, async_sessionmaker

from switch_core.config import SwitchConfig
from switch_core.logging_context import LogContextFilter
from switch_core.observability.catalogue import (
    AGENTS_CONNECTED,
    BRIDGES_CRASHED,
    BRIDGES_RUNNING,
    CONSUMERS_RUNNING,
    DB_POOL_IN_USE,
    DB_POOL_OVERFLOW,
    DB_POOL_SIZE,
    DELIVERY_CACHE_BYTES,
    DELIVERY_CACHE_ROOMS,
    RUNTIME_STARTS,
)
from switch_core.observability.db_server import DbServerSampler
from switch_core.observability.exporter import MetricsExporter
from switch_core.observability.health import (
    HealthMonitor,
    agent_event_listener_check,
    bridges_check,
    database_check,
    message_listener_check,
    session_activity_listener_check,
    signal_bus_check,
)
from switch_core.observability.logs import (
    DEFAULT_BATCH_SIZE,
    DEFAULT_QUEUE_CAPACITY,
    LogExporter,
    OtlpLogHandler,
)
from switch_core.observability.metrics import (
    GaugeReading,
    MetricsRegistry,
    install,
    uninstall,
)
from switch_core.observability.otlp import OtlpClient, OtlpResource
from switch_core.observability.pool import PoolStats
from switch_core.observability.runtime import (
    EventLoopLag,
    GcPauses,
    RuntimeMetrics,
    log_unreadable_sources,
    watch_event_loop,
)
from switch_core.transport.room_cache import RoomCacheStats
from switch_core.version import switch_core_commit, switch_core_repository_url

logger = logging.getLogger(__name__)

# Matched to the kubelet's probe period; longer and a fault is reported one
# probe later than it could have been.
HEALTH_REFRESH_INTERVAL_SECONDS = 10.0

# Far shorter than the metric interval: a metric interval is a bucket, a log is
# read by someone looking at an incident now.
LOG_EXPORT_INTERVAL_SECONDS = 5.0


@dataclass(frozen=True)
class RuntimeProbes:
    """Live state, as callables over whatever holds it.

    Callables rather than the services, so this module does not depend on half
    the application and a test can supply the numbers without a bridge.
    """

    listener_connected: Callable[[], bool]
    session_activity_listener_connected: Callable[[], bool]
    agent_event_listener_connected: Callable[[], bool]
    signal_bus_connected: Callable[[], bool]
    bridges_running: Callable[[], int]
    bridges_running_by_platform: Callable[[], Mapping[str, int]]
    bridges_configured: Callable[[], int]
    # (platform, display name) per bridge whose task has died.
    bridges_crashed: Callable[[], Sequence[tuple[str, str]]]
    consumers_running: Callable[[], int]
    # Agents per (transport, client); see
    # `AgentConnectionRegistry.live_agents_by_transport`.
    agents_connected: Callable[[], Mapping[tuple[str, str], int]]
    # None when the engine's pool does not keep these — see
    # :mod:`switch_core.observability.pool`.
    pool_stats: Callable[[], PoolStats | None]
    room_cache_stats: Callable[[], RoomCacheStats]


@dataclass
class Observability:
    """What the server needs to hold on to: the monitor, and how to stop."""

    monitor: HealthMonitor
    lag: EventLoopLag
    _tasks: list[asyncio.Task[None]]
    _http_client: httpx.AsyncClient | None
    _log_handler: OtlpLogHandler | None
    _gc_pauses: GcPauses | None

    async def aclose(self) -> None:
        # Detached first, so shutdown logging is not queued for an exporter
        # about to stop draining it.
        if self._log_handler is not None:
            logging.getLogger().removeHandler(self._log_handler)
        if self._gc_pauses is not None:
            self._gc_pauses.uninstall()
        for task in self._tasks:
            task.cancel()
        for task in self._tasks:
            try:
                await task
            except asyncio.CancelledError:
                pass
            except Exception:
                logger.exception("An observability task failed on shutdown.")
        if self._http_client is not None:
            await self._http_client.aclose()
        uninstall()


def _state_readings(probes: RuntimeProbes) -> Callable[[], Iterator[GaugeReading]]:
    # Every (transport, client) pair reported so far: one that empties out is
    # reported as 0, or the dashboard would keep showing its last count.
    seen: set[tuple[str, str]] = {("websocket", "unknown")}
    # Every bridge reported down so far: once it runs again it reads 0, so its
    # alert recovers instead of going quiet.
    crashed_seen: set[tuple[str, str]] = set()

    def readings() -> Iterator[GaugeReading]:
        connected = probes.agents_connected()
        seen.update(connected)
        for transport, client in sorted(seen):
            yield GaugeReading(
                AGENTS_CONNECTED,
                float(connected.get((transport, client), 0)),
                {"transport": transport, "client": client},
            )
        yield GaugeReading(CONSUMERS_RUNNING, float(probes.consumers_running()), {})
        for platform, running in probes.bridges_running_by_platform().items():
            yield GaugeReading(
                BRIDGES_RUNNING,
                float(running),
                {"bridge": "collaboration", "platform": platform},
            )
        crashed = set(probes.bridges_crashed())
        crashed_seen.update(crashed)
        for platform, name in sorted(crashed_seen):
            yield GaugeReading(
                BRIDGES_CRASHED,
                1.0 if (platform, name) in crashed else 0.0,
                {"platform": platform, "bridge_name": name},
            )

        stats = probes.pool_stats()
        if stats is not None:
            yield GaugeReading(DB_POOL_IN_USE, float(stats.in_use), {})
            yield GaugeReading(DB_POOL_SIZE, float(stats.size), {})
            yield GaugeReading(DB_POOL_OVERFLOW, float(stats.overflow), {})

        cache = probes.room_cache_stats()
        yield GaugeReading(DELIVERY_CACHE_BYTES, float(cache.bytes), {})
        yield GaugeReading(DELIVERY_CACHE_ROOMS, float(cache.rooms), {})

    return readings


def start_observability(
    config: SwitchConfig,
    version: str | None,
    session_factory: async_sessionmaker,
    probes: RuntimeProbes,
    process_id: str,
    db_server_engine: Callable[[], AsyncEngine] | None = None,
) -> Observability:
    """Install the registry, start the loops, and hand back the handle.

    Called once, from the server's lifespan. `db_server_engine` builds the
    engine the database sampler takes its one connection from; it must not be
    the application's pooled engine. `process_id` names this process on
    everything it reports (`process_identity.py`).
    """
    # With another replica to take the traffic, a process that would answer
    # late is better out of service; with one, it is the only one there is.
    listeners_gate = config.core_replicas > 1
    monitor = HealthMonitor(
        checks=[
            database_check(session_factory),
            message_listener_check(probes.listener_connected, listeners_gate),
            session_activity_listener_check(
                probes.session_activity_listener_connected, listeners_gate
            ),
            agent_event_listener_check(
                probes.agent_event_listener_connected, listeners_gate
            ),
            signal_bus_check(probes.signal_bus_connected, listeners_gate),
            bridges_check(probes.bridges_running, probes.bridges_configured),
        ],
        interval_seconds=HEALTH_REFRESH_INTERVAL_SECONDS,
    )
    lag = EventLoopLag()
    tasks: list[asyncio.Task[None]] = [
        asyncio.create_task(monitor.run_forever(), name="health-monitor")
    ]

    if not config.observability_enabled:
        logger.info(
            "No OTLP_ENDPOINT is configured, so nothing is reported off this "
            "server. Health checks still run and /health/ready still answers."
        )
        return Observability(
            monitor=monitor,
            lag=lag,
            _tasks=tasks,
            _http_client=None,
            _log_handler=None,
            _gc_pauses=None,
        )

    registry = MetricsRegistry()
    install(registry)
    registry.increment(RUNTIME_STARTS, {})

    monitor.install(registry)
    gc_pauses = GcPauses()
    gc_pauses.install()
    RuntimeMetrics(lag, gc_pauses).install(registry)
    tasks.append(asyncio.create_task(watch_event_loop(lag), name="event-loop-probe"))
    registry.register_observer(_state_readings(probes))
    log_unreadable_sources()

    http_client = httpx.AsyncClient()
    client = OtlpClient(
        base_endpoint=str(config.otlp_endpoint),
        timeout_seconds=config.otlp_timeout_seconds,
        headers=config.otlp_header_map,
        client=http_client,
    )
    resource = OtlpResource(
        service_name=config.service_name,
        service_version=version,
        environment=config.environment,
        deployment_id=str(config.deployment_id),
        commit_sha=switch_core_commit(),
        repository_url=switch_core_repository_url(),
        instance_id=process_id,
        pod_name=config.pod_name or socket.gethostname(),
    )

    if config.otlp_metrics_enabled:
        exporter = MetricsExporter(
            registry=registry,
            client=client,
            resource=resource,
            interval_seconds=config.otlp_export_interval_seconds,
        )
        tasks.append(
            asyncio.create_task(exporter.run_forever(), name="metrics-exporter")
        )
        if db_server_engine is not None:
            sampler = DbServerSampler(db_server_engine)
            registry.register_observer(sampler.readings)
            tasks.append(
                asyncio.create_task(sampler.run_forever(), name="db-server-sampler")
            )
        logger.info(
            "Reporting metrics to %s every %.0fs as service %r.",
            client.url_for("metrics"),
            config.otlp_export_interval_seconds,
            config.service_name,
        )
    else:
        logger.warning(
            "OTLP_ENDPOINT is set but OTLP_METRICS_ENABLED is false, so no "
            "metrics are being reported."
        )

    log_handler: OtlpLogHandler | None = None
    if config.otlp_logs_enabled:
        log_handler = OtlpLogHandler(capacity=DEFAULT_QUEUE_CAPACITY)
        # The same filter the stderr handler carries; without it a shipped
        # record has no tenant, request or agent on it.
        log_handler.addFilter(LogContextFilter(config.tenant_id))
        logging.getLogger().addHandler(log_handler)
        tasks.append(
            asyncio.create_task(
                LogExporter(
                    handler=log_handler,
                    client=client,
                    resource=resource,
                    interval_seconds=LOG_EXPORT_INTERVAL_SECONDS,
                    batch_size=DEFAULT_BATCH_SIZE,
                ).run_forever(),
                name="log-exporter",
            )
        )
        logger.info(
            "Also shipping logs to %s. They continue to be written to this "
            "container's output, which remains the primary copy.",
            client.url_for("logs"),
        )

    return Observability(
        monitor=monitor,
        lag=lag,
        _tasks=tasks,
        _http_client=http_client,
        _log_handler=log_handler,
        _gc_pauses=gc_pauses,
    )