Skip to content

Enterprise agent platform for building, deploying, and operating AI teammates.

11.0 KB
Raw Blame
from __future__ import annotations

from contextlib import nullcontext

import pytest
from fastapi import FastAPI

from xagent.web import app as app_module
from xagent.web import runtime_telemetry


@pytest.fixture
def metric_reader(monkeypatch):
    from opentelemetry.sdk.metrics import MeterProvider
    from opentelemetry.sdk.metrics.export import InMemoryMetricReader

    from xagent.core import runtime_performance as metrics

    reader = InMemoryMetricReader()
    provider = MeterProvider(metric_readers=[reader], shutdown_on_exit=False)
    monkeypatch.setattr(
        metrics,
        "runtime_performance",
        metrics.RuntimePerformanceTelemetry(provider.get_meter("test")),
    )
    yield reader
    provider.shutdown()


def collected_metrics(reader):
    data = reader.get_metrics_data()
    assert data is not None
    return {
        m.name: m
        for r in data.resource_metrics
        for s in r.scope_metrics
        for m in s.metrics
    }


@pytest.mark.asyncio
async def test_login_rejection_emits_metrics(monkeypatch, metric_reader):
    from fastapi import HTTPException

    from xagent.web.api import auth

    monkeypatch.setattr(auth, "get_user_by_login_identifier", lambda *_: None)
    with pytest.raises(HTTPException) as error:
        await auth.login(
            auth.LoginRequest(username="unknown", password="test"), db=nullcontext
        )
    assert error.value.status_code == 401
    metrics = collected_metrics(metric_reader)
    point = metrics["xagent.auth.login.requests"].data.data_points[0]
    assert point.value == 1
    assert point.attributes["outcome"] == "rejected"
    assert metrics["xagent.auth.login.total.duration"].data.data_points[0].count == 1


@pytest.mark.asyncio
async def test_broadcast_records_one_payload_for_multiple_connections(
    metric_reader, caplog
):
    from unittest.mock import AsyncMock

    from xagent.web.api.websocket import ConnectionManager

    manager = ConnectionManager()
    sockets = [AsyncMock(), AsyncMock()]
    for socket in sockets:
        manager.register_connection(socket, 42)
    await manager.broadcast_to_task({"type": "diagnostic"}, 42)
    metrics = collected_metrics(metric_reader)
    assert metrics["xagent.websocket.payload.size"].data.data_points[0].count == 1
    assert metrics["xagent.websocket.messages.sent"].data.data_points[0].value == 2
    assert metrics["xagent.websocket.broadcast.fanout"].data.data_points[0].sum == 2
    with pytest.raises(TypeError):
        await manager.broadcast_to_task({"type": "diagnostic", "bad": object()}, 42)
    assert manager.connections_for_task(42) == sockets
    assert "Failed to serialize WebSocket broadcast" in caplog.text


@pytest.mark.asyncio
async def test_database_trace_handler_emits_write_outcome(monkeypatch, metric_reader):
    # Own the runtime as well as the writer: changing the backend flag alone
    # still reads process-global DB state left by unrelated app/config tests.
    from xagent.core.agent.trace import TASK_START_GENERAL, TraceEvent
    from xagent.web.services import trace_database, trace_handlers
    from xagent.web.services.trace_database import TraceDatabaseRuntime
    from xagent.web.services.trace_handlers import DatabaseTraceHandler

    runtime = TraceDatabaseRuntime(None, use_async=False, limit=1)
    monkeypatch.setattr(trace_handlers, "get_trace_database_runtime", lambda: runtime)
    # Reproduce the invalid ambient engine from the CI failure. This unit test
    # must not inspect it or open any database connection.
    monkeypatch.setattr(trace_database, "get_engine", lambda: object())
    handler = DatabaseTraceHandler(42)
    saved = []
    monkeypatch.setattr(
        handler, "_sync_save_to_database", lambda event, **_: saved.append(event)
    )
    event = TraceEvent(
        event_type=TASK_START_GENERAL, task_id="42", data={}, require_persisted=True
    )
    try:
        await handler.handle_event(event)
    finally:
        await runtime.close()
    assert saved == [event]
    point = collected_metrics(metric_reader)[
        "xagent.trace.database.writes"
    ].data.data_points[0]
    assert point.value == 1
    assert point.attributes["outcome"] == "succeeded"


@pytest.mark.asyncio
async def test_async_trace_records_one_serialization_and_write(
    tmp_path, monkeypatch, metric_reader
):
    from sqlalchemy import create_engine
    from sqlalchemy.orm import Session

    from xagent.core.agent.trace import TASK_START_GENERAL, TraceEvent
    from xagent.web.models.database import Base
    from xagent.web.models.task import Task
    from xagent.web.models.user import User
    from xagent.web.services import trace_handlers
    from xagent.web.services.trace_database import TraceDatabaseRuntime

    source = create_engine(f"sqlite:///{tmp_path / 'metrics.db'}")
    Base.metadata.create_all(source)
    with Session(source) as db:
        user = User(username="trace-metrics", password_hash="unused")
        db.add(user)
        db.flush()
        task = Task(user_id=user.id, title="Metrics")
        db.add(task)
        db.flush()
        task_id = task.id
        db.commit()
    runtime = TraceDatabaseRuntime(source, use_async=True, limit=4)
    monkeypatch.setattr(trace_handlers, "get_trace_database_runtime", lambda: runtime)
    try:
        await trace_handlers.DatabaseTraceHandler(task_id).handle_event(
            TraceEvent(
                TASK_START_GENERAL,
                task_id=str(task_id),
                data={},
                require_persisted=True,
            )
        )
        metrics = collected_metrics(metric_reader)
        assert (
            metrics["xagent.trace.database.serialization.duration"]
            .data.data_points[0]
            .count
            == 1
        )
        assert (
            metrics["xagent.trace.database.preparation.duration"]
            .data.data_points[0]
            .count
            == 1
        )
        writes = metrics["xagent.trace.database.writes"].data.data_points
        assert len(writes) == 1
        assert writes[0].value == 1
        assert writes[0].attributes["outcome"] == "succeeded"
    finally:
        await runtime.close()
        source.dispose()


@pytest.mark.asyncio
async def test_websocket_trace_handler_emits_metrics(monkeypatch, metric_reader):
    from unittest.mock import AsyncMock

    from xagent.core.agent.trace import TASK_START_GENERAL, TraceEvent
    from xagent.web.api.websocket import ConnectionManager
    from xagent.web.services import task_event_trace_handler as ws_trace_handlers
    from xagent.web.services import task_events

    handler = ws_trace_handlers.TaskEventTraceHandler(42)
    handler._task_description_loaded = True
    monkeypatch.setattr(handler, "_has_prior_user_message_turn", lambda *_: False)
    # This case measures what the handler emits for an event it actually
    # publishes, so it needs an audience: without one the handler returns
    # before the metrics below are ever recorded.
    audience_manager = ConnectionManager()
    audience_manager.register_connection(AsyncMock(), 42)
    monkeypatch.setattr(
        task_events, "_task_audience_probe", audience_manager.has_connections_for_task
    )
    broadcast = AsyncMock()
    monkeypatch.setattr(ws_trace_handlers, "publish_task_event", broadcast)
    await handler.handle_event(
        TraceEvent(event_type=TASK_START_GENERAL, task_id="42", data={})
    )
    broadcast.assert_awaited_once()
    metrics = collected_metrics(metric_reader)
    trace_points = metrics["xagent.websocket.trace.events"].data.data_points
    broadcast_point = next(
        point for point in trace_points if point.attributes["outcome"] == "broadcast"
    )
    assert broadcast_point.value == 1
    assert (
        metrics["xagent.websocket.trace_serialization.duration"]
        .data.data_points[0]
        .count
        == 1
    )


def test_runtime_performance_monitor_stays_off_without_exporter(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    test_app = FastAPI()
    monkeypatch.setattr(
        runtime_telemetry,
        "initialize_runtime_performance_telemetry",
        lambda: False,
    )

    app_module.start_runtime_performance_monitor(test_app)

    assert test_app.state.runtime_performance_task is None


@pytest.mark.asyncio
async def test_runtime_performance_monitor_registers_gauges_once_and_stops(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    test_app = FastAPI()
    registered: list[str] = []
    shutdown_calls: list[bool] = []
    monkeypatch.setattr(
        runtime_telemetry,
        "initialize_runtime_performance_telemetry",
        lambda: True,
    )
    monkeypatch.setattr(
        runtime_telemetry,
        "register_observable_gauge",
        lambda metric, *_args, **_kwargs: registered.append(metric),
    )
    monkeypatch.setattr(
        runtime_telemetry,
        "shutdown_runtime_performance_telemetry",
        lambda: shutdown_calls.append(True),
    )

    app_module.start_runtime_performance_monitor(test_app)
    first_task = test_app.state.runtime_performance_task
    app_module.start_runtime_performance_monitor(test_app)

    assert test_app.state.runtime_performance_task is first_task
    assert registered == [
        "xagent.agent_tasks.running",
        "xagent.agent_tasks.resuming",
        "xagent.websocket.connections",
    ]

    await app_module.stop_runtime_performance_monitor(test_app)
    assert test_app.state.runtime_performance_task is None
    assert first_task.cancelled()
    assert shutdown_calls == [True]


def test_process_telemetry_that_fails_to_start_shuts_down_and_returns_none(
    monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture
) -> None:
    """A worker relies on this never raising: telemetry must not block startup."""
    shutdown_calls: list[bool] = []
    monkeypatch.setattr(
        runtime_telemetry, "initialize_runtime_performance_telemetry", lambda: True
    )

    def broken_gauge(*_args, **_kwargs):
        raise RuntimeError("meter rejected the gauge")

    monkeypatch.setattr(runtime_telemetry, "register_observable_gauge", broken_gauge)
    monkeypatch.setattr(
        runtime_telemetry,
        "shutdown_runtime_performance_telemetry",
        lambda: shutdown_calls.append(True),
    )

    assert runtime_telemetry.start_process_runtime_telemetry() is None
    assert shutdown_calls == [True]
    assert "Could not start runtime performance monitor" in caplog.text


@pytest.mark.asyncio
async def test_worker_telemetry_registers_no_websocket_gauge(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    registered: list[str] = []
    monkeypatch.setattr(
        runtime_telemetry, "initialize_runtime_performance_telemetry", lambda: True
    )
    monkeypatch.setattr(
        runtime_telemetry,
        "register_observable_gauge",
        lambda metric, *_args, **_kwargs: registered.append(metric),
    )
    monkeypatch.setattr(
        runtime_telemetry, "shutdown_runtime_performance_telemetry", lambda: None
    )

    task = runtime_telemetry.start_process_runtime_telemetry()
    try:
        assert registered == [
            "xagent.agent_tasks.running",
            "xagent.agent_tasks.resuming",
        ]
    finally:
        await runtime_telemetry.stop_process_runtime_telemetry(task)