Skip to content

BrainAPI is a knowledge graph–powered AI memory layer that transforms unstructured data into structured knowledge, enabling intelligent search, recommendations, and contextual memory for AI agents and applications.

Add modular tracing SLA tracker

Cursor Agente4b918ef0e7c
diff --git a/src/lib/tracing/__init__.py b/src/lib/tracing/__init__.py
new file mode 100644
index 0000000..62f84bf
--- /dev/null
+++ b/src/lib/tracing/__init__.py
@@ -0,0 +1,17 @@
+from src.lib.tracing.events import (
+ TraceEvent,
+ TraceEventType,
+ TraceSeverity,
+)
+from src.lib.tracing.subscribers import TraceSubscriber
+from src.lib.tracing.tracker import LocalTraceQueue, TraceTracker, trace_tracker
+
+__all__ = [
+ "LocalTraceQueue",
+ "TraceEvent",
+ "TraceEventType",
+ "TraceSeverity",
+ "TraceSubscriber",
+ "TraceTracker",
+ "trace_tracker",
+]
diff --git a/src/lib/tracing/events.py b/src/lib/tracing/events.py
new file mode 100644
index 0000000..21e9410
--- /dev/null
+++ b/src/lib/tracing/events.py
@@ -0,0 +1,58 @@
+from dataclasses import dataclass, field
+from enum import Enum
+import time
+from typing import Any
+
+
+class TraceEventType(str, Enum):
+ ERROR = "error"
+ EXCEPTION = "exception"
+ DOWNTIME = "downtime"
+ EXPENSIVE_LOOP = "expensive_loop"
+ SLA_BREACH = "sla_breach"
+ LATENCY = "latency"
+
+
+class TraceSeverity(str, Enum):
+ INFO = "info"
+ WARNING = "warning"
+ ERROR = "error"
+ CRITICAL = "critical"
+
+
+@dataclass(frozen=True)
+class TraceEvent:
+ event_type: TraceEventType
+ name: str
+ severity: TraceSeverity = TraceSeverity.INFO
+ service: str | None = None
+ operation: str | None = None
+ tenant_id: str | None = None
+ trace_id: str | None = None
+ duration_ms: float | None = None
+ threshold_ms: float | None = None
+ status_code: int | None = None
+ error_type: str | None = None
+ message: str | None = None
+ metadata: dict[str, Any] = field(default_factory=dict)
+ stack_trace: str | None = None
+ created_at: float = field(default_factory=time.time)
+
+ def to_dict(self) -> dict[str, Any]:
+ return {
+ "event_type": self.event_type.value,
+ "name": self.name,
+ "severity": self.severity.value,
+ "service": self.service,
+ "operation": self.operation,
+ "tenant_id": self.tenant_id,
+ "trace_id": self.trace_id,
+ "duration_ms": self.duration_ms,
+ "threshold_ms": self.threshold_ms,
+ "status_code": self.status_code,
+ "error_type": self.error_type,
+ "message": self.message,
+ "metadata": dict(self.metadata),
+ "stack_trace": self.stack_trace,
+ "created_at": self.created_at,
+ }
diff --git a/src/lib/tracing/middleware.py b/src/lib/tracing/middleware.py
new file mode 100644
index 0000000..667fa2c
--- /dev/null
+++ b/src/lib/tracing/middleware.py
@@ -0,0 +1,124 @@
+import os
+import time
+from uuid import uuid4
+
+from starlette.types import ASGIApp, Message, Receive, Scope, Send
+
+from src.lib.tracing.events import TraceEventType, TraceSeverity
+from src.lib.tracing.tracker import tracer
+
+
+class TraceMiddleware:
+ def __init__(
+ self,
+ app: ASGIApp,
+ service_name: str,
+ slow_request_ms: float | None = None,
+ ):
+ self.app = app
+ self.service_name = service_name
+ self.slow_request_ms = (
+ slow_request_ms
+ if slow_request_ms is not None
+ else float(os.getenv("TRACE_SLOW_REQUEST_MS", "1000"))
+ )
+
+ async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None:
+ if scope["type"] not in ("http", "websocket"):
+ await self.app(scope, receive, send)
+ return
+
+ started_at = time.perf_counter()
+ status_code: int | None = None
+ tenant_id = self._tenant_id(scope)
+ trace_id = self._header(scope, "x-trace-id") or str(uuid4())
+ tokens = tracer.set_context(trace_id=trace_id, tenant_id=tenant_id)
+
+ async def send_wrapper(message: Message) -> None:
+ nonlocal status_code
+ if message["type"] == "http.response.start":
+ status_code = message.get("status")
+ await send(message)
+
+ try:
+ await self.app(scope, receive, send_wrapper)
+ except Exception as exc:
+ duration_ms = (time.perf_counter() - started_at) * 1000
+ tracer.exception(
+ name=self._operation(scope),
+ exception=exc,
+ service=self.service_name,
+ operation=self._operation(scope),
+ duration_ms=duration_ms,
+ status_code=status_code,
+ threshold_ms=self.slow_request_ms,
+ metadata=self._scope_metadata(scope, status_code),
+ )
+ tracer.reset_context(tokens)
+ raise
+
+ duration_ms = (time.perf_counter() - started_at) * 1000
+ operation = self._operation(scope)
+ if status_code is not None and status_code >= 500:
+ tracer.error(
+ name=operation,
+ service=self.service_name,
+ operation=operation,
+ message=f"{self.service_name} returned HTTP {status_code}",
+ severity=TraceSeverity.ERROR,
+ duration_ms=duration_ms,
+ status_code=status_code,
+ threshold_ms=self.slow_request_ms,
+ metadata=self._scope_metadata(scope, status_code),
+ )
+
+ if self.slow_request_ms is not None and duration_ms >= self.slow_request_ms:
+ tracer.publish(
+ TraceEventType.SLA_BREACH,
+ name=operation,
+ severity=TraceSeverity.WARNING,
+ service=self.service_name,
+ operation=operation,
+ duration_ms=duration_ms,
+ threshold_ms=self.slow_request_ms,
+ status_code=status_code,
+ message=f"{operation} exceeded SLA threshold",
+ metadata=self._scope_metadata(scope, status_code),
+ )
+ tracer.reset_context(tokens)
+
+ def _operation(self, scope: Scope) -> str:
+ method = scope.get("method", scope["type"])
+ return f"{method} {scope.get('path', '')}".strip()
+
+ def _scope_metadata(self, scope: Scope, status_code: int | None) -> dict[str, str]:
+ metadata = {
+ "service": self.service_name,
+ "protocol": scope["type"],
+ "path": str(scope.get("path", "")),
+ }
+ if status_code is not None:
+ metadata["status_code"] = str(status_code)
+ return metadata
+
+ def _tenant_id(self, scope: Scope) -> str | None:
+ return (
+ self._header(scope, "x-brain-id")
+ or self._query_param(scope, "brain_id")
+ or self._header(scope, "x-tenant-id")
+ )
+
+ def _header(self, scope: Scope, name: str) -> str | None:
+ encoded_name = name.lower().encode()
+ for header_name, value in scope.get("headers", []):
+ if header_name.lower() == encoded_name:
+ return value.decode(errors="replace").rstrip() or None
+ return None
+
+ def _query_param(self, scope: Scope, name: str) -> str | None:
+ raw_query = scope.get("query_string", b"").decode(errors="replace")
+ for item in raw_query.split("&"):
+ key, _, value = item.partition("=")
+ if key == name and value:
+ return value.rstrip()
+ return None
diff --git a/src/lib/tracing/subscribers.py b/src/lib/tracing/subscribers.py
new file mode 100644
index 0000000..805d0ac
--- /dev/null
+++ b/src/lib/tracing/subscribers.py
@@ -0,0 +1,15 @@
+from abc import ABC, abstractmethod
+from typing import Iterable
+
+from src.lib.tracing.events import TraceEvent
+
+
+class TraceSubscriber(ABC):
+ @abstractmethod
+ def publish(self, event: TraceEvent) -> None:
+ raise NotImplementedError("publish method not implemented")
+
+ def publish_many(self, events: Iterable[TraceEvent]) -> None:
+ for event in events:
+ self.publish(event)
+
diff --git a/src/lib/tracing/tracker.py b/src/lib/tracing/tracker.py
new file mode 100644
index 0000000..134da40
--- /dev/null
+++ b/src/lib/tracing/tracker.py
@@ -0,0 +1,363 @@
+import asyncio
+import inspect
+import logging
+import os
+import time
+import traceback
+from contextlib import contextmanager
+from contextvars import ContextVar
+from typing import Any, Iterator, Optional
+
+from src.lib.tracing.events import TraceEvent, TraceEventType, TraceSeverity
+from src.lib.tracing.subscribers import TraceSubscriber
+
+logger = logging.getLogger(__name__)
+
+_trace_id_var: ContextVar[Optional[str]] = ContextVar("trace_id", default=None)
+_tenant_id_var: ContextVar[Optional[str]] = ContextVar("tenant_id", default=None)
+
+
+class LocalTraceQueue:
+ def __init__(self, max_size: int = 10000):
+ self._queue: asyncio.Queue[TraceEvent] = asyncio.Queue(maxsize=max_size)
+
+ @property
+ def size(self) -> int:
+ return self._queue.qsize()
+
+ def put_nowait(self, event: TraceEvent) -> bool:
+ try:
+ self._queue.put_nowait(event)
+ return True
+ except asyncio.QueueFull:
+ return False
+
+ async def get(self) -> TraceEvent:
+ return await self._queue.get()
+
+ def task_done(self) -> None:
+ self._queue.task_done()
+
+ def drain(self, limit: Optional[int] = None) -> list[TraceEvent]:
+ events: list[TraceEvent] = []
+ while not self._queue.empty() and (limit is None or len(events) < limit):
+ events.append(self._queue.get_nowait())
+ self._queue.task_done()
+ return events
+
+
+class TraceTracker:
+ def __init__(
+ self,
+ *,
+ queue: Optional[LocalTraceQueue] = None,
+ max_queue_size: Optional[int] = None,
+ slow_operation_ms: Optional[float] = None,
+ downtime_ms: Optional[float] = None,
+ expensive_loop_iterations: Optional[int] = None,
+ enabled: Optional[bool] = None,
+ ):
+ self.queue = queue or LocalTraceQueue(
+ max_queue_size
+ if max_queue_size is not None
+ else int(os.getenv("TRACE_QUEUE_MAX_SIZE", "10000"))
+ )
+ self.slow_operation_ms = (
+ slow_operation_ms
+ if slow_operation_ms is not None
+ else float(os.getenv("TRACE_SLOW_OPERATION_MS", "1000"))
+ )
+ self.downtime_ms = (
+ downtime_ms
+ if downtime_ms is not None
+ else float(os.getenv("TRACE_DOWNTIME_MS", "5000"))
+ )
+ self.expensive_loop_iterations = (
+ expensive_loop_iterations
+ if expensive_loop_iterations is not None
+ else int(os.getenv("TRACE_EXPENSIVE_LOOP_ITERATIONS", "10000"))
+ )
+ self.enabled = (
+ enabled
+ if enabled is not None
+ else os.getenv("TRACE_TRACKER_ENABLED", "true") == "true"
+ )
+ self._subscribers: list[TraceSubscriber] = []
+ self._subscriber_tasks: list[asyncio.Task] = []
+
+ @property
+ def default_sla_ms(self) -> float:
+ return self.slow_operation_ms
+
+ def set_context(
+ self, *, trace_id: Optional[str] = None, tenant_id: Optional[str] = None
+ ):
+ trace_token = _trace_id_var.set(trace_id) if trace_id is not None else None
+ tenant_token = _tenant_id_var.set(tenant_id) if tenant_id is not None else None
+ return trace_token, tenant_token
+
+ def reset_context(self, tokens) -> None:
+ trace_token, tenant_token = tokens
+ if trace_token is not None:
+ _trace_id_var.reset(trace_token)
+ if tenant_token is not None:
+ _tenant_id_var.reset(tenant_token)
+
+ def subscribe(self, subscriber: TraceSubscriber) -> None:
+ self._subscribers.append(subscriber)
+
+ def clear_subscribers(self) -> None:
+ self._subscribers.clear()
+
+ def publish(
+ self,
+ event_type: TraceEventType | str,
+ name: str,
+ *,
+ severity: TraceSeverity | str = TraceSeverity.INFO,
+ tenant_id: Optional[str] = None,
+ trace_id: Optional[str] = None,
+ service: Optional[str] = None,
+ operation: Optional[str] = None,
+ duration_ms: Optional[float] = None,
+ threshold_ms: Optional[float] = None,
+ status_code: Optional[int] = None,
+ message: Optional[str] = None,
+ metadata: Optional[dict[str, Any]] = None,
+ exception: Optional[BaseException] = None,
+ ) -> Optional[TraceEvent]:
+ if not self.enabled:
+ return None
+ event_trace_id = trace_id or _trace_id_var.get()
+ event_message = message or (str(exception) if exception else None)
+ event = TraceEvent(
+ event_type=TraceEventType(event_type),
+ severity=TraceSeverity(severity),
+ name=name,
+ service=service,
+ operation=operation,
+ tenant_id=tenant_id or _tenant_id_var.get(),
+ trace_id=event_trace_id,
+ duration_ms=duration_ms,
+ threshold_ms=threshold_ms,
+ status_code=status_code,
+ error_type=type(exception).__name__ if exception else None,
+ message=event_message,
+ metadata=dict(metadata or {}),
+ stack_trace=(
+ "".join(traceback.format_exception(exception)) if exception else None
+ ),
+ )
+ if not self.queue.put_nowait(event):
+ logger.warning("Trace queue is full; dropping event %s", event.name)
+ return None
+ return event
+
+ def exception(
+ self,
+ name: str,
+ exception: BaseException,
+ *,
+ service: Optional[str] = None,
+ operation: Optional[str] = None,
+ tenant_id: Optional[str] = None,
+ trace_id: Optional[str] = None,
+ duration_ms: Optional[float] = None,
+ threshold_ms: Optional[float] = None,
+ status_code: Optional[int] = None,
+ metadata: Optional[dict[str, Any]] = None,
+ ) -> Optional[TraceEvent]:
+ return self.publish(
+ TraceEventType.EXCEPTION,
+ name,
+ severity=TraceSeverity.ERROR,
+ service=service,
+ operation=operation,
+ tenant_id=tenant_id,
+ trace_id=trace_id,
+ duration_ms=duration_ms,
+ threshold_ms=threshold_ms,
+ status_code=status_code,
+ metadata=metadata,
+ exception=exception,
+ )
+
+ def error(
+ self,
+ name: str,
+ *,
+ service: Optional[str] = None,
+ operation: Optional[str] = None,
+ tenant_id: Optional[str] = None,
+ trace_id: Optional[str] = None,
+ severity: TraceSeverity | str = TraceSeverity.ERROR,
+ duration_ms: Optional[float] = None,
+ threshold_ms: Optional[float] = None,
+ status_code: Optional[int] = None,
+ message: Optional[str] = None,
+ metadata: Optional[dict[str, Any]] = None,
+ ) -> Optional[TraceEvent]:
+ return self.publish(
+ TraceEventType.ERROR,
+ name,
+ severity=severity,
+ service=service,
+ operation=operation,
+ tenant_id=tenant_id,
+ trace_id=trace_id,
+ duration_ms=duration_ms,
+ threshold_ms=threshold_ms,
+ status_code=status_code,
+ message=message,
+ metadata=metadata,
+ )
+
+ def downtime(
+ self,
+ name: str,
+ duration_ms: float,
+ *,
+ service: Optional[str] = None,
+ operation: Optional[str] = None,
+ tenant_id: Optional[str] = None,
+ trace_id: Optional[str] = None,
+ metadata: Optional[dict[str, Any]] = None,
+ ) -> Optional[TraceEvent]:
+ return self.publish(
+ TraceEventType.DOWNTIME,
+ name,
+ severity=TraceSeverity.CRITICAL,
+ service=service,
+ operation=operation,
+ tenant_id=tenant_id,
+ trace_id=trace_id,
+ duration_ms=duration_ms,
+ metadata=metadata,
+ )
+
+ def expensive_loop(
+ self,
+ name: str,
+ iterations: int,
+ *,
+ service: Optional[str] = None,
+ operation: Optional[str] = None,
+ tenant_id: Optional[str] = None,
+ trace_id: Optional[str] = None,
+ metadata: Optional[dict[str, Any]] = None,
+ ) -> Optional[TraceEvent]:
+ if iterations < self.expensive_loop_iterations:
+ return None
+ return self.publish(
+ TraceEventType.EXPENSIVE_LOOP,
+ name,
+ severity=TraceSeverity.WARNING,
+ service=service,
+ operation=operation,
+ tenant_id=tenant_id,
+ trace_id=trace_id,
+ metadata={**(metadata or {}), "iterations": iterations},
+ )
+
+ @contextmanager
+ def span(
+ self,
+ name: str,
+ *,
+ tenant_id: Optional[str] = None,
+ trace_id: Optional[str] = None,
+ metadata: Optional[dict[str, Any]] = None,
+ slow_operation_ms: Optional[float] = None,
+ ) -> Iterator[None]:
+ tokens = self.set_context(trace_id=trace_id, tenant_id=tenant_id)
+ started_at = time.perf_counter()
+ try:
+ yield
+ except Exception as exc:
+ duration_ms = (time.perf_counter() - started_at) * 1000
+ self.exception(
+ name,
+ exc,
+ metadata={**(metadata or {}), "duration_ms": duration_ms},
+ )
+ raise
+ finally:
+ duration_ms = (time.perf_counter() - started_at) * 1000
+ threshold = (
+ slow_operation_ms
+ if slow_operation_ms is not None
+ else self.slow_operation_ms
+ )
+ if duration_ms >= threshold:
+ self.publish(
+ TraceEventType.SLA_BREACH,
+ name,
+ severity=TraceSeverity.WARNING,
+ duration_ms=duration_ms,
+ threshold_ms=threshold,
+ metadata=metadata,
+ )
+ self.reset_context(tokens)
+
+ def track_loop(
+ self,
+ name: str,
+ iterable,
+ *,
+ tenant_id: Optional[str] = None,
+ trace_id: Optional[str] = None,
+ metadata: Optional[dict[str, Any]] = None,
+ threshold: Optional[int] = None,
+ ):
+ count = 0
+ for item in iterable:
+ count += 1
+ yield item
+ if count >= (threshold or self.expensive_loop_iterations):
+ self.expensive_loop(
+ name,
+ count,
+ tenant_id=tenant_id,
+ trace_id=trace_id,
+ metadata=metadata,
+ )
+
+ async def dispatch_once(self) -> TraceEvent:
+ event = await self.queue.get()
+ try:
+ for subscriber in list(self._subscribers):
+ result = subscriber.handle(event)
+ if inspect.isawaitable(result):
+ await result
+ finally:
+ self.queue.task_done()
+ return event
+
+ async def subscribe_forever(self) -> None:
+ while True:
+ await self.dispatch_once()
+
+ def start_subscribers(self) -> None:
+ if not self._subscribers or self._subscriber_tasks:
+ return
+ try:
+ loop = asyncio.get_running_loop()
+ except RuntimeError:
+ return
+ self._subscriber_tasks = [
+ loop.create_task(self.subscribe_forever()) for _ in self._subscribers
+ ]
+
+ async def stop_subscribers(self) -> None:
+ for task in self._subscriber_tasks:
+ task.cancel()
+ for task in self._subscriber_tasks:
+ try:
+ await task
+ except asyncio.CancelledError:
+ pass
+ self._subscriber_tasks = []
+
+
+trace_tracker = TraceTracker()
+tracer = trace_tracker
diff --git a/src/services/api/app.py b/src/services/api/app.py
index cd50784..82adefa 100644
--- a/src/services/api/app.py
+++ b/src/services/api/app.py
@@ -20,6 +20,7 @@ from src.services.api.routes.model import model_router
from src.services.api.routes.retrieve import retrieve_router
from src.services.api.routes.system import system_router
from src.services.api.routes.tasks import tasks_router
+from src.lib.tracing.middleware import TraceMiddleware
logger = logging.getLogger("brainapi.plugins")
@@ -109,6 +110,7 @@ app = FastAPI(
redirect_slashes=False,
)
+app.add_middleware(TraceMiddleware, service_name="brainapi-api")
app.add_middleware(BrainPATMiddleware)
app.add_middleware(BrainMiddleware)
app.add_middleware(
diff --git a/src/services/mcp/app.py b/src/services/mcp/app.py
index 01f19e2..5bccb0f 100644
--- a/src/services/mcp/app.py
+++ b/src/services/mcp/app.py
@@ -11,6 +11,7 @@ from starlette.routing import Route
_project_root = Path(__file__).resolve().parent.parent.parent.parent
dotenv.load_dotenv(_project_root / ".env")
+from src.lib.tracing.middleware import TraceMiddleware
from src.services.mcp.main import auth_token_var, mcp
PLUGINS_DIR = Path(os.getenv("PLUGINS_DIR", str(_project_root / "plugins")))
@@ -118,6 +119,9 @@ _custom_routes = [
]
app = Starlette(
routes=_custom_routes + list(_mcp_app.routes),
- middleware=[Middleware(AuthContextMiddleware)],
+ middleware=[
+ Middleware(AuthContextMiddleware),
+ Middleware(TraceMiddleware, service_name="brainapi-mcp"),
+ ],
lifespan=_mcp_app.router.lifespan_context,
)
diff --git a/tests/test_tracing_sla_tracker.py b/tests/test_tracing_sla_tracker.py
new file mode 100644
index 0000000..ed47b2a
--- /dev/null
+++ b/tests/test_tracing_sla_tracker.py
@@ -0,0 +1,164 @@
+import asyncio
+import unittest
+
+from starlette.applications import Starlette
+from starlette.responses import JSONResponse, PlainTextResponse
+from starlette.middleware import Middleware
+from starlette.routing import Route
+from starlette.testclient import TestClient
+
+from src.lib.tracing import (
+ LocalTraceQueue,
+ TraceEvent,
+ TraceEventType,
+ TraceSeverity,
+ TraceSubscriber,
+ TraceTracker,
+)
+from src.lib.tracing.middleware import TraceMiddleware
+from src.lib.tracing import middleware as tracing_middleware
+
+
+class MemorySubscriber(TraceSubscriber):
+ def __init__(self):
+ self.events = []
+
+ def handle(self, event: TraceEvent) -> None:
+ self.events.append(event)
+
+
+class TraceTrackerTests(unittest.TestCase):
+ def test_publish_records_event_with_tenant_and_trace_context(self):
+ tracker = TraceTracker(queue=LocalTraceQueue(), enabled=True)
+ tokens = tracker.set_context(trace_id="trace-1", tenant_id="tenant-1")
+ try:
+ event = tracker.error(
+ "operation.failed",
+ service="api",
+ operation="GET /demo",
+ message="failed",
+ )
+ finally:
+ tracker.reset_context(tokens)
+
+ self.assertIsNotNone(event)
+ drained = tracker.queue.drain()
+ self.assertEqual(len(drained), 1)
+ self.assertEqual(drained[0].event_type, TraceEventType.ERROR)
+ self.assertEqual(drained[0].severity, TraceSeverity.ERROR)
+ self.assertEqual(drained[0].tenant_id, "tenant-1")
+ self.assertEqual(drained[0].trace_id, "trace-1")
+ self.assertEqual(drained[0].service, "api")
+ self.assertEqual(drained[0].operation, "GET /demo")
+
+ def test_span_records_exception_and_sla_breach(self):
+ tracker = TraceTracker(
+ queue=LocalTraceQueue(),
+ enabled=True,
+ slow_operation_ms=0,
+ )
+
+ with self.assertRaises(ValueError):
+ with tracker.span("failing-span", tenant_id="brain1"):
+ raise ValueError("boom")
+
+ events = tracker.queue.drain()
+ self.assertEqual([event.event_type for event in events], [
+ TraceEventType.EXCEPTION,
+ TraceEventType.SLA_BREACH,
+ ])
+ self.assertEqual(events[0].error_type, "ValueError")
+ self.assertIn("boom", events[0].message)
+ self.assertEqual(events[0].tenant_id, "brain1")
+ self.assertIsNotNone(events[0].stack_trace)
+
+ def test_track_loop_records_expensive_loop_when_threshold_is_met(self):
+ tracker = TraceTracker(
+ queue=LocalTraceQueue(),
+ enabled=True,
+ expensive_loop_iterations=3,
+ )
+
+ self.assertEqual(list(tracker.track_loop("loop", range(3))), [0, 1, 2])
+
+ events = tracker.queue.drain()
+ self.assertEqual(len(events), 1)
+ self.assertEqual(events[0].event_type, TraceEventType.EXPENSIVE_LOOP)
+ self.assertEqual(events[0].metadata["iterations"], 3)
+
+ def test_dispatch_once_sends_event_to_subscriber(self):
+ async def run_test():
+ tracker = TraceTracker(queue=LocalTraceQueue(), enabled=True)
+ subscriber = MemorySubscriber()
+ tracker.subscribe(subscriber)
+ event = tracker.error("error")
+
+ dispatched = await tracker.dispatch_once()
+
+ self.assertIs(dispatched, event)
+ self.assertEqual(subscriber.events, [event])
+
+ asyncio.run(run_test())
+
+
+class TraceMiddlewareTests(unittest.TestCase):
+ def setUp(self):
+ self.original_tracer = tracing_middleware.tracer
+ self.tracker = TraceTracker(queue=LocalTraceQueue(), enabled=True)
+ tracing_middleware.tracer = self.tracker
+
+ def tearDown(self):
+ tracing_middleware.tracer = self.original_tracer
+
+ def test_middleware_records_server_errors_and_tenant_id(self):
+ async def error_route(_request):
+ return JSONResponse({"detail": "bad"}, status_code=503)
+
+ app = Starlette(
+ routes=[Route("/bad", error_route)],
+ middleware=[
+ Middleware(
+ TraceMiddleware,
+ service_name="test-api",
+ slow_request_ms=100000,
+ )
+ ],
+ )
+
+ response = TestClient(app).get("/bad", headers={"X-Brain-ID": "tenant-a"})
+
+ self.assertEqual(response.status_code, 503)
+ events = self.tracker.queue.drain()
+ self.assertEqual(len(events), 1)
+ self.assertEqual(events[0].event_type, TraceEventType.ERROR)
+ self.assertEqual(events[0].tenant_id, "tenant-a")
+ self.assertEqual(events[0].service, "test-api")
+ self.assertEqual(events[0].status_code, 503)
+
+ def test_middleware_records_sla_breach(self):
+ async def ok_route(_request):
+ return PlainTextResponse("ok")
+
+ app = Starlette(
+ routes=[Route("/ok", ok_route)],
+ middleware=[
+ Middleware(
+ TraceMiddleware,
+ service_name="test-api",
+ slow_request_ms=0,
+ )
+ ],
+ )
+
+ response = TestClient(app).get("/ok?brain_id=tenant-b")
+
+ self.assertEqual(response.status_code, 200)
+ events = self.tracker.queue.drain()
+ self.assertEqual(len(events), 1)
+ self.assertEqual(events[0].event_type, TraceEventType.SLA_BREACH)
+ self.assertEqual(events[0].tenant_id, "tenant-b")
+ self.assertEqual(events[0].operation, "GET /ok")
+
+
+if __name__ == "__main__":
+ unittest.main()