Enterprise agent platform for building, deploying, and operating AI teammates.
from __future__ import annotations
import asyncio
import threading
from collections.abc import Iterator
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from typing import Any
from unittest.mock import MagicMock
import pytest
from opentelemetry.proto.collector.metrics.v1.metrics_service_pb2 import (
ExportMetricsServiceRequest,
)
from opentelemetry.sdk.metrics import MeterProvider
from opentelemetry.sdk.metrics.export import InMemoryMetricReader
import xagent.core.runtime_performance as runtime_metrics
from xagent.core.agent.trace import TASK_START_GENERAL, TraceEvent, TraceHandler, Tracer
from xagent.core.runtime_performance import (
METER_NAME,
RuntimePerformanceTelemetry,
_histogram_views,
monitor_event_loop_lag,
run_in_thread_with_telemetry,
runtime_performance,
)
@pytest.mark.parametrize(
"endpoint",
[
"localhost:4318",
"http://localhost:4318?secret=value",
"http://localhost:4318/#fragment",
],
)
def test_invalid_endpoint_disables_telemetry(monkeypatch, caplog, endpoint):
monkeypatch.setenv("XAGENT_RUNTIME_TELEMETRY_ENABLED", "true")
monkeypatch.setenv("XAGENT_OTEL_EXPORTER_OTLP_METRICS_ENDPOINT", endpoint)
assert runtime_metrics.initialize_runtime_performance_telemetry() is False
assert "runtime telemetry is disabled" in caplog.text
assert endpoint not in caplog.text
def test_meter_failure_does_not_mask_business_result_or_error(monkeypatch):
meter = MagicMock()
meter.create_counter.side_effect = RuntimeError("broken meter")
meter.create_histogram.return_value.record.side_effect = RuntimeError(
"broken histogram"
)
monkeypatch.setattr(
runtime_metrics, "runtime_performance", RuntimePerformanceTelemetry(meter)
)
runtime_metrics.increment_counter("xagent.test")
with runtime_metrics.observe_duration("xagent.test.duration"):
result = 42
assert result == 42
with pytest.raises(ValueError, match="business error"):
with runtime_metrics.observe_duration("xagent.test.duration"):
raise ValueError("business error")
@pytest.fixture
def otel_backend() -> Iterator[
tuple[RuntimePerformanceTelemetry, InMemoryMetricReader]
]:
reader = InMemoryMetricReader()
provider = MeterProvider(
metric_readers=[reader],
views=_histogram_views(),
shutdown_on_exit=False,
)
telemetry = RuntimePerformanceTelemetry(provider.get_meter(METER_NAME))
try:
yield telemetry, reader
finally:
provider.shutdown()
def _metrics(reader: InMemoryMetricReader) -> dict[str, Any]:
data = reader.get_metrics_data()
assert data is not None
return {
metric.name: metric
for resource_metrics in data.resource_metrics
for scope_metrics in resource_metrics.scope_metrics
for metric in scope_metrics.metrics
}
@pytest.mark.parametrize(
"metric_name",
[
"xagent.trace.dispatch.duration",
"xagent.trace.database.admission_wait.duration",
"xagent.trace.database.preparation.duration",
],
)
def test_records_counter_and_explicit_bucket_histogram_in_otel(
otel_backend: tuple[RuntimePerformanceTelemetry, InMemoryMetricReader],
metric_name: str,
) -> None:
telemetry, reader = otel_backend
telemetry.increment(
"xagent.trace.events",
attributes={"event.type": "task_start_general"},
)
telemetry.observe(
metric_name,
6.0,
unit="ms",
)
metrics = _metrics(reader)
counter_point = metrics["xagent.trace.events"].data.data_points[0]
histogram_point = metrics[metric_name].data.data_points[0]
assert counter_point.value == 1
assert counter_point.attributes == {"event.type": "task_start_general"}
assert histogram_point.count == 1
assert histogram_point.sum == 6.0
assert 10.0 in histogram_point.explicit_bounds
assert (
histogram_point.bucket_counts[histogram_point.explicit_bounds.index(10.0)] == 1
)
def test_execution_event_read_histograms_use_duration_and_row_buckets(
otel_backend: tuple[RuntimePerformanceTelemetry, InMemoryMetricReader],
) -> None:
telemetry, reader = otel_backend
telemetry.observe("xagent.execution_event.read.duration", 42.0, unit="ms")
telemetry.observe("xagent.execution_event.read.rows", 30_000, unit="{row}")
metrics = _metrics(reader)
duration = metrics["xagent.execution_event.read.duration"].data.data_points[0]
rows = metrics["xagent.execution_event.read.rows"].data.data_points[0]
assert tuple(duration.explicit_bounds) == runtime_metrics._DURATION_BUCKETS_MS
# The generic count buckets stop at 1024; reads reach tens of thousands.
assert tuple(rows.explicit_bounds) == (
0.0,
1.0,
10.0,
50.0,
100.0,
250.0,
500.0,
1_000.0,
2_500.0,
5_000.0,
10_000.0,
25_000.0,
50_000.0,
100_000.0,
)
assert rows.bucket_counts[rows.explicit_bounds.index(50_000.0)] == 1
assert rows.sum == 30_000
def test_attribute_allowlist_and_cardinality_guard_exclude_sensitive_values(
otel_backend: tuple[RuntimePerformanceTelemetry, InMemoryMetricReader],
) -> None:
telemetry, reader = otel_backend
for index in range(100):
telemetry.increment(
"xagent.auth.login.requests",
attributes={
"outcome": f"outcome-{index}",
"task_id": "sensitive-task-id",
},
)
metric = _metrics(reader)["xagent.auth.login.requests"]
points = metric.data.data_points
by_outcome = {point.attributes["outcome"]: point.value for point in points}
assert len(points) == 65
assert by_outcome["other"] == 36
assert "sensitive-task-id" not in str(metric)
assert all("task_id" not in point.attributes for point in points)
def test_observable_gauge_reads_current_value(
otel_backend: tuple[RuntimePerformanceTelemetry, InMemoryMetricReader],
) -> None:
telemetry, reader = otel_backend
current = {"value": 2}
telemetry.register_observable_gauge(
"xagent.agent_tasks.running",
lambda: current["value"],
unit="{task}",
description="Current locally running agent tasks",
)
first_point = _metrics(reader)["xagent.agent_tasks.running"].data.data_points[0]
assert first_point.value == 2
current["value"] = 5
second_point = _metrics(reader)["xagent.agent_tasks.running"].data.data_points[0]
assert second_point.value == 5
def test_otlp_http_exporter_posts_metrics_to_configured_collector(
monkeypatch: pytest.MonkeyPatch,
) -> None:
requests: list[tuple[str, str, bytes]] = []
received = threading.Event()
class CollectorHandler(BaseHTTPRequestHandler):
def do_POST(self) -> None: # noqa: N802
body = self.rfile.read(int(self.headers.get("Content-Length", "0")))
requests.append((self.path, self.headers.get("Content-Type", ""), body))
self.send_response(200)
self.end_headers()
received.set()
def log_message(self, _format: str, *_args: object) -> None:
return
server = ThreadingHTTPServer(("127.0.0.1", 0), CollectorHandler)
server_thread = threading.Thread(target=server.serve_forever, daemon=True)
server_thread.start()
endpoint = f"http://127.0.0.1:{server.server_port}/v1/metrics"
monkeypatch.setattr(
runtime_metrics,
"get_runtime_telemetry_enabled",
lambda: True,
)
monkeypatch.setattr(
runtime_metrics,
"get_otel_metrics_endpoint",
lambda: endpoint,
)
monkeypatch.setattr(
runtime_metrics,
"get_otel_export_interval_milliseconds",
lambda: 50,
)
monkeypatch.setattr(
runtime_metrics,
"get_otel_service_name",
lambda: "xagent-test",
)
runtime_metrics.shutdown_runtime_performance_telemetry()
try:
assert runtime_metrics.initialize_runtime_performance_telemetry() is True
runtime_metrics.increment_counter("xagent.trace.events")
assert received.wait(timeout=2.0)
path, content_type, body = requests[0]
export_request = ExportMetricsServiceRequest.FromString(body)
metric_names = {
metric.name
for resource_metrics in export_request.resource_metrics
for scope_metrics in resource_metrics.scope_metrics
for metric in scope_metrics.metrics
}
resource_attributes = {
attribute.key: attribute.value.string_value
for resource_metrics in export_request.resource_metrics
for attribute in resource_metrics.resource.attributes
}
assert path == "/v1/metrics"
assert content_type == "application/x-protobuf"
assert "xagent.trace.events" in metric_names
assert resource_attributes["service.name"] == "xagent-test"
finally:
runtime_metrics.shutdown_runtime_performance_telemetry()
server.shutdown()
server.server_close()
server_thread.join(timeout=2.0)
@pytest.mark.asyncio
async def test_thread_helper_exports_queue_execution_and_total_histograms() -> None:
reader = InMemoryMetricReader()
provider = MeterProvider(metric_readers=[reader], shutdown_on_exit=False)
runtime_performance.bind_meter(provider.get_meter(METER_NAME))
try:
result = await run_in_thread_with_telemetry("unit_test", lambda: 42)
assert result == 42
metrics = _metrics(reader)
for metric_name in (
"xagent.thread_pool.queue_wait.duration",
"xagent.thread_pool.execution.duration",
"xagent.thread_pool.total.duration",
):
point = metrics[metric_name].data.data_points[0]
assert point.count == 1
assert point.attributes == {"operation": "unit_test"}
finally:
runtime_performance.disable()
provider.shutdown()
@pytest.mark.asyncio
async def test_event_loop_monitor_and_tracer_export_without_payload_content() -> None:
class RecordingHandler(TraceHandler):
async def handle_event(self, event: TraceEvent) -> None:
await asyncio.sleep(0)
reader = InMemoryMetricReader()
provider = MeterProvider(metric_readers=[reader], shutdown_on_exit=False)
runtime_performance.bind_meter(provider.get_meter(METER_NAME))
monitor = asyncio.create_task(monitor_event_loop_lag(0.001))
try:
tracer = Tracer()
tracer.add_handler(RecordingHandler())
await tracer.trace_event(
TASK_START_GENERAL,
task_id="sensitive-task-id",
data={"prompt": "sensitive prompt body"},
)
await asyncio.sleep(0.01)
monitor.cancel()
with pytest.raises(asyncio.CancelledError):
await monitor
metrics = _metrics(reader)
assert "xagent.event_loop.lag" in metrics
assert "xagent.trace.events" in metrics
assert "xagent.trace.handler.duration" in metrics
serialized_metrics = str(metrics)
assert "sensitive-task-id" not in serialized_metrics
assert "sensitive prompt body" not in serialized_metrics
finally:
if not monitor.done():
monitor.cancel()
with pytest.raises(asyncio.CancelledError):
await monitor
runtime_performance.disable()
provider.shutdown()
def _attribute_keys_in_source() -> dict[str, list[str]]:
"""``attributes=`` keys passed anywhere under ``src/xagent``.
Covers a dict literal passed directly and a name bound to a dict literal
in the same module (``attributes = {...}`` then ``attributes=attributes``).
Keys built any other way, such as from a parameter or with
``dict(...)``, are not seen.
"""
import ast
from pathlib import Path
keys: dict[str, list[str]] = {}
# src/xagent: this module lives in src/xagent/core.
root = Path(runtime_metrics.__file__).resolve().parents[1]
for path in root.rglob("*.py"):
tree = ast.parse(path.read_text(encoding="utf-8"))
bound: dict[str, list[ast.Dict]] = {}
for node in ast.walk(tree):
if (
isinstance(node, ast.Assign)
and isinstance(node.value, ast.Dict)
and len(node.targets) == 1
and isinstance(node.targets[0], ast.Name)
):
bound.setdefault(node.targets[0].id, []).append(node.value)
for node in ast.walk(tree):
if not isinstance(node, ast.Call):
continue
for keyword in node.keywords:
if keyword.arg != "attributes":
continue
if isinstance(keyword.value, ast.Dict):
dicts = [keyword.value]
elif isinstance(keyword.value, ast.Name):
dicts = bound.get(keyword.value.id, [])
else:
continue
for literal in dicts:
for key in literal.keys:
if isinstance(key, ast.Constant) and isinstance(key.value, str):
keys.setdefault(key.value, []).append(
f"{path.relative_to(root)}:{key.lineno}"
)
return keys
def test_every_metric_attribute_key_in_source_is_exported() -> None:
"""An unlisted key is dropped silently at export, so list it or rename it."""
keys = _attribute_keys_in_source()
# The scan itself works, for both a literal and a name bound to one.
assert any(
site.startswith("web/services/task_admission_observation.py")
for site in keys["outcome"]
)
unlisted = {
key: sites
for key, sites in keys.items()
if key not in runtime_metrics._ALLOWED_ATTRIBUTE_KEYS
}
assert unlisted == {}