Enterprise agent platform for building, deploying, and operating AI teammates.
from __future__ import annotations
from contextlib import nullcontext
import pytest
from fastapi import FastAPI
from xagent.web import app as app_module
from xagent.web import runtime_telemetry
@pytest.fixture
def metric_reader(monkeypatch):
from opentelemetry.sdk.metrics import MeterProvider
from opentelemetry.sdk.metrics.export import InMemoryMetricReader
from xagent.core import runtime_performance as metrics
reader = InMemoryMetricReader()
provider = MeterProvider(metric_readers=[reader], shutdown_on_exit=False)
monkeypatch.setattr(
metrics,
"runtime_performance",
metrics.RuntimePerformanceTelemetry(provider.get_meter("test")),
)
yield reader
provider.shutdown()
def collected_metrics(reader):
data = reader.get_metrics_data()
assert data is not None
return {
m.name: m
for r in data.resource_metrics
for s in r.scope_metrics
for m in s.metrics
}
@pytest.mark.asyncio
async def test_login_rejection_emits_metrics(monkeypatch, metric_reader):
from fastapi import HTTPException
from xagent.web.api import auth
monkeypatch.setattr(auth, "get_user_by_login_identifier", lambda *_: None)
with pytest.raises(HTTPException) as error:
await auth.login(
auth.LoginRequest(username="unknown", password="test"), db=nullcontext
)
assert error.value.status_code == 401
metrics = collected_metrics(metric_reader)
point = metrics["xagent.auth.login.requests"].data.data_points[0]
assert point.value == 1
assert point.attributes["outcome"] == "rejected"
assert metrics["xagent.auth.login.total.duration"].data.data_points[0].count == 1
@pytest.mark.asyncio
async def test_broadcast_records_one_payload_for_multiple_connections(
metric_reader, caplog
):
from unittest.mock import AsyncMock
from xagent.web.api.websocket import ConnectionManager
manager = ConnectionManager()
sockets = [AsyncMock(), AsyncMock()]
for socket in sockets:
manager.register_connection(socket, 42)
await manager.broadcast_to_task({"type": "diagnostic"}, 42)
metrics = collected_metrics(metric_reader)
assert metrics["xagent.websocket.payload.size"].data.data_points[0].count == 1
assert metrics["xagent.websocket.messages.sent"].data.data_points[0].value == 2
assert metrics["xagent.websocket.broadcast.fanout"].data.data_points[0].sum == 2
with pytest.raises(TypeError):
await manager.broadcast_to_task({"type": "diagnostic", "bad": object()}, 42)
assert manager.connections_for_task(42) == sockets
assert "Failed to serialize WebSocket broadcast" in caplog.text
@pytest.mark.asyncio
async def test_database_trace_handler_emits_write_outcome(monkeypatch, metric_reader):
# Own the runtime as well as the writer: changing the backend flag alone
# still reads process-global DB state left by unrelated app/config tests.
from xagent.core.agent.trace import TASK_START_GENERAL, TraceEvent
from xagent.web.services import trace_database, trace_handlers
from xagent.web.services.trace_database import TraceDatabaseRuntime
from xagent.web.services.trace_handlers import DatabaseTraceHandler
runtime = TraceDatabaseRuntime(None, use_async=False, limit=1)
monkeypatch.setattr(trace_handlers, "get_trace_database_runtime", lambda: runtime)
# Reproduce the invalid ambient engine from the CI failure. This unit test
# must not inspect it or open any database connection.
monkeypatch.setattr(trace_database, "get_engine", lambda: object())
handler = DatabaseTraceHandler(42)
saved = []
monkeypatch.setattr(
handler, "_sync_save_to_database", lambda event, **_: saved.append(event)
)
event = TraceEvent(
event_type=TASK_START_GENERAL, task_id="42", data={}, require_persisted=True
)
try:
await handler.handle_event(event)
finally:
await runtime.close()
assert saved == [event]
point = collected_metrics(metric_reader)[
"xagent.trace.database.writes"
].data.data_points[0]
assert point.value == 1
assert point.attributes["outcome"] == "succeeded"
@pytest.mark.asyncio
async def test_async_trace_records_one_serialization_and_write(
tmp_path, monkeypatch, metric_reader
):
from sqlalchemy import create_engine
from sqlalchemy.orm import Session
from xagent.core.agent.trace import TASK_START_GENERAL, TraceEvent
from xagent.web.models.database import Base
from xagent.web.models.task import Task
from xagent.web.models.user import User
from xagent.web.services import trace_handlers
from xagent.web.services.trace_database import TraceDatabaseRuntime
source = create_engine(f"sqlite:///{tmp_path / 'metrics.db'}")
Base.metadata.create_all(source)
with Session(source) as db:
user = User(username="trace-metrics", password_hash="unused")
db.add(user)
db.flush()
task = Task(user_id=user.id, title="Metrics")
db.add(task)
db.flush()
task_id = task.id
db.commit()
runtime = TraceDatabaseRuntime(source, use_async=True, limit=4)
monkeypatch.setattr(trace_handlers, "get_trace_database_runtime", lambda: runtime)
try:
await trace_handlers.DatabaseTraceHandler(task_id).handle_event(
TraceEvent(
TASK_START_GENERAL,
task_id=str(task_id),
data={},
require_persisted=True,
)
)
metrics = collected_metrics(metric_reader)
assert (
metrics["xagent.trace.database.serialization.duration"]
.data.data_points[0]
.count
== 1
)
assert (
metrics["xagent.trace.database.preparation.duration"]
.data.data_points[0]
.count
== 1
)
writes = metrics["xagent.trace.database.writes"].data.data_points
assert len(writes) == 1
assert writes[0].value == 1
assert writes[0].attributes["outcome"] == "succeeded"
finally:
await runtime.close()
source.dispose()
@pytest.mark.asyncio
async def test_websocket_trace_handler_emits_metrics(monkeypatch, metric_reader):
from unittest.mock import AsyncMock
from xagent.core.agent.trace import TASK_START_GENERAL, TraceEvent
from xagent.web.api.websocket import ConnectionManager
from xagent.web.services import task_event_trace_handler as ws_trace_handlers
from xagent.web.services import task_events
handler = ws_trace_handlers.TaskEventTraceHandler(42)
handler._task_description_loaded = True
monkeypatch.setattr(handler, "_has_prior_user_message_turn", lambda *_: False)
# This case measures what the handler emits for an event it actually
# publishes, so it needs an audience: without one the handler returns
# before the metrics below are ever recorded.
audience_manager = ConnectionManager()
audience_manager.register_connection(AsyncMock(), 42)
monkeypatch.setattr(
task_events, "_task_audience_probe", audience_manager.has_connections_for_task
)
broadcast = AsyncMock()
monkeypatch.setattr(ws_trace_handlers, "publish_task_event", broadcast)
await handler.handle_event(
TraceEvent(event_type=TASK_START_GENERAL, task_id="42", data={})
)
broadcast.assert_awaited_once()
metrics = collected_metrics(metric_reader)
trace_points = metrics["xagent.websocket.trace.events"].data.data_points
broadcast_point = next(
point for point in trace_points if point.attributes["outcome"] == "broadcast"
)
assert broadcast_point.value == 1
assert (
metrics["xagent.websocket.trace_serialization.duration"]
.data.data_points[0]
.count
== 1
)
def test_runtime_performance_monitor_stays_off_without_exporter(
monkeypatch: pytest.MonkeyPatch,
) -> None:
test_app = FastAPI()
monkeypatch.setattr(
runtime_telemetry,
"initialize_runtime_performance_telemetry",
lambda: False,
)
app_module.start_runtime_performance_monitor(test_app)
assert test_app.state.runtime_performance_task is None
@pytest.mark.asyncio
async def test_runtime_performance_monitor_registers_gauges_once_and_stops(
monkeypatch: pytest.MonkeyPatch,
) -> None:
test_app = FastAPI()
registered: list[str] = []
shutdown_calls: list[bool] = []
monkeypatch.setattr(
runtime_telemetry,
"initialize_runtime_performance_telemetry",
lambda: True,
)
monkeypatch.setattr(
runtime_telemetry,
"register_observable_gauge",
lambda metric, *_args, **_kwargs: registered.append(metric),
)
monkeypatch.setattr(
runtime_telemetry,
"shutdown_runtime_performance_telemetry",
lambda: shutdown_calls.append(True),
)
app_module.start_runtime_performance_monitor(test_app)
first_task = test_app.state.runtime_performance_task
app_module.start_runtime_performance_monitor(test_app)
assert test_app.state.runtime_performance_task is first_task
assert registered == [
"xagent.agent_tasks.running",
"xagent.agent_tasks.resuming",
"xagent.websocket.connections",
]
await app_module.stop_runtime_performance_monitor(test_app)
assert test_app.state.runtime_performance_task is None
assert first_task.cancelled()
assert shutdown_calls == [True]
def test_process_telemetry_that_fails_to_start_shuts_down_and_returns_none(
monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture
) -> None:
"""A worker relies on this never raising: telemetry must not block startup."""
shutdown_calls: list[bool] = []
monkeypatch.setattr(
runtime_telemetry, "initialize_runtime_performance_telemetry", lambda: True
)
def broken_gauge(*_args, **_kwargs):
raise RuntimeError("meter rejected the gauge")
monkeypatch.setattr(runtime_telemetry, "register_observable_gauge", broken_gauge)
monkeypatch.setattr(
runtime_telemetry,
"shutdown_runtime_performance_telemetry",
lambda: shutdown_calls.append(True),
)
assert runtime_telemetry.start_process_runtime_telemetry() is None
assert shutdown_calls == [True]
assert "Could not start runtime performance monitor" in caplog.text
@pytest.mark.asyncio
async def test_worker_telemetry_registers_no_websocket_gauge(
monkeypatch: pytest.MonkeyPatch,
) -> None:
registered: list[str] = []
monkeypatch.setattr(
runtime_telemetry, "initialize_runtime_performance_telemetry", lambda: True
)
monkeypatch.setattr(
runtime_telemetry,
"register_observable_gauge",
lambda metric, *_args, **_kwargs: registered.append(metric),
)
monkeypatch.setattr(
runtime_telemetry, "shutdown_runtime_performance_telemetry", lambda: None
)
task = runtime_telemetry.start_process_runtime_telemetry()
try:
assert registered == [
"xagent.agent_tasks.running",
"xagent.agent_tasks.resuming",
]
finally:
await runtime_telemetry.stop_process_runtime_telemetry(task)