BrainAPI is a knowledge graph–powered AI memory layer that transforms unstructured data into structured knowledge, enabling intelligent search, recommendations, and contextual memory for AI agents and applications.
Python ai-agentsai-memoryknowledge-graphknowledge-managementknowledge-representationreccomendation-system
Add modular tracing SLA tracker
Cursor Agente4b918ef0e7c
diff --git a/src/lib/tracing/__init__.py b/src/lib/tracing/__init__.pynew file mode 100644index 0000000..62f84bf--- /dev/null+++ b/src/lib/tracing/__init__.py@@ -0,0 +1,17 @@+from src.lib.tracing.events import (+ TraceEvent,+ TraceEventType,+ TraceSeverity,+)+from src.lib.tracing.subscribers import TraceSubscriber+from src.lib.tracing.tracker import LocalTraceQueue, TraceTracker, trace_tracker++__all__ = [+ "LocalTraceQueue",+ "TraceEvent",+ "TraceEventType",+ "TraceSeverity",+ "TraceSubscriber",+ "TraceTracker",+ "trace_tracker",+]diff --git a/src/lib/tracing/events.py b/src/lib/tracing/events.pynew file mode 100644index 0000000..21e9410--- /dev/null+++ b/src/lib/tracing/events.py@@ -0,0 +1,58 @@+from dataclasses import dataclass, field+from enum import Enum+import time+from typing import Any+++class TraceEventType(str, Enum):+ ERROR = "error"+ EXCEPTION = "exception"+ DOWNTIME = "downtime"+ EXPENSIVE_LOOP = "expensive_loop"+ SLA_BREACH = "sla_breach"+ LATENCY = "latency"+++class TraceSeverity(str, Enum):+ INFO = "info"+ WARNING = "warning"+ ERROR = "error"+ CRITICAL = "critical"+++@dataclass(frozen=True)+class TraceEvent:+ event_type: TraceEventType+ name: str+ severity: TraceSeverity = TraceSeverity.INFO+ service: str | None = None+ operation: str | None = None+ tenant_id: str | None = None+ trace_id: str | None = None+ duration_ms: float | None = None+ threshold_ms: float | None = None+ status_code: int | None = None+ error_type: str | None = None+ message: str | None = None+ metadata: dict[str, Any] = field(default_factory=dict)+ stack_trace: str | None = None+ created_at: float = field(default_factory=time.time)++ def to_dict(self) -> dict[str, Any]:+ return {+ "event_type": self.event_type.value,+ "name": self.name,+ "severity": self.severity.value,+ "service": self.service,+ "operation": self.operation,+ "tenant_id": self.tenant_id,+ "trace_id": self.trace_id,+ "duration_ms": self.duration_ms,+ "threshold_ms": self.threshold_ms,+ "status_code": self.status_code,+ "error_type": self.error_type,+ "message": self.message,+ "metadata": dict(self.metadata),+ "stack_trace": self.stack_trace,+ "created_at": self.created_at,+ }diff --git a/src/lib/tracing/middleware.py b/src/lib/tracing/middleware.pynew file mode 100644index 0000000..667fa2c--- /dev/null+++ b/src/lib/tracing/middleware.py@@ -0,0 +1,124 @@+import os+import time+from uuid import uuid4++from starlette.types import ASGIApp, Message, Receive, Scope, Send++from src.lib.tracing.events import TraceEventType, TraceSeverity+from src.lib.tracing.tracker import tracer+++class TraceMiddleware:+ def __init__(+ self,+ app: ASGIApp,+ service_name: str,+ slow_request_ms: float | None = None,+ ):+ self.app = app+ self.service_name = service_name+ self.slow_request_ms = (+ slow_request_ms+ if slow_request_ms is not None+ else float(os.getenv("TRACE_SLOW_REQUEST_MS", "1000"))+ )++ async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None:+ if scope["type"] not in ("http", "websocket"):+ await self.app(scope, receive, send)+ return++ started_at = time.perf_counter()+ status_code: int | None = None+ tenant_id = self._tenant_id(scope)+ trace_id = self._header(scope, "x-trace-id") or str(uuid4())+ tokens = tracer.set_context(trace_id=trace_id, tenant_id=tenant_id)++ async def send_wrapper(message: Message) -> None:+ nonlocal status_code+ if message["type"] == "http.response.start":+ status_code = message.get("status")+ await send(message)++ try:+ await self.app(scope, receive, send_wrapper)+ except Exception as exc:+ duration_ms = (time.perf_counter() - started_at) * 1000+ tracer.exception(+ name=self._operation(scope),+ exception=exc,+ service=self.service_name,+ operation=self._operation(scope),+ duration_ms=duration_ms,+ status_code=status_code,+ threshold_ms=self.slow_request_ms,+ metadata=self._scope_metadata(scope, status_code),+ )+ tracer.reset_context(tokens)+ raise++ duration_ms = (time.perf_counter() - started_at) * 1000+ operation = self._operation(scope)+ if status_code is not None and status_code >= 500:+ tracer.error(+ name=operation,+ service=self.service_name,+ operation=operation,+ message=f"{self.service_name} returned HTTP {status_code}",+ severity=TraceSeverity.ERROR,+ duration_ms=duration_ms,+ status_code=status_code,+ threshold_ms=self.slow_request_ms,+ metadata=self._scope_metadata(scope, status_code),+ )++ if self.slow_request_ms is not None and duration_ms >= self.slow_request_ms:+ tracer.publish(+ TraceEventType.SLA_BREACH,+ name=operation,+ severity=TraceSeverity.WARNING,+ service=self.service_name,+ operation=operation,+ duration_ms=duration_ms,+ threshold_ms=self.slow_request_ms,+ status_code=status_code,+ message=f"{operation} exceeded SLA threshold",+ metadata=self._scope_metadata(scope, status_code),+ )+ tracer.reset_context(tokens)++ def _operation(self, scope: Scope) -> str:+ method = scope.get("method", scope["type"])+ return f"{method} {scope.get('path', '')}".strip()++ def _scope_metadata(self, scope: Scope, status_code: int | None) -> dict[str, str]:+ metadata = {+ "service": self.service_name,+ "protocol": scope["type"],+ "path": str(scope.get("path", "")),+ }+ if status_code is not None:+ metadata["status_code"] = str(status_code)+ return metadata++ def _tenant_id(self, scope: Scope) -> str | None:+ return (+ self._header(scope, "x-brain-id")+ or self._query_param(scope, "brain_id")+ or self._header(scope, "x-tenant-id")+ )++ def _header(self, scope: Scope, name: str) -> str | None:+ encoded_name = name.lower().encode()+ for header_name, value in scope.get("headers", []):+ if header_name.lower() == encoded_name:+ return value.decode(errors="replace").rstrip() or None+ return None++ def _query_param(self, scope: Scope, name: str) -> str | None:+ raw_query = scope.get("query_string", b"").decode(errors="replace")+ for item in raw_query.split("&"):+ key, _, value = item.partition("=")+ if key == name and value:+ return value.rstrip()+ return Nonediff --git a/src/lib/tracing/subscribers.py b/src/lib/tracing/subscribers.pynew file mode 100644index 0000000..805d0ac--- /dev/null+++ b/src/lib/tracing/subscribers.py@@ -0,0 +1,15 @@+from abc import ABC, abstractmethod+from typing import Iterable++from src.lib.tracing.events import TraceEvent+++class TraceSubscriber(ABC):+ @abstractmethod+ def publish(self, event: TraceEvent) -> None:+ raise NotImplementedError("publish method not implemented")++ def publish_many(self, events: Iterable[TraceEvent]) -> None:+ for event in events:+ self.publish(event)+diff --git a/src/lib/tracing/tracker.py b/src/lib/tracing/tracker.pynew file mode 100644index 0000000..134da40--- /dev/null+++ b/src/lib/tracing/tracker.py@@ -0,0 +1,363 @@+import asyncio+import inspect+import logging+import os+import time+import traceback+from contextlib import contextmanager+from contextvars import ContextVar+from typing import Any, Iterator, Optional++from src.lib.tracing.events import TraceEvent, TraceEventType, TraceSeverity+from src.lib.tracing.subscribers import TraceSubscriber++logger = logging.getLogger(__name__)++_trace_id_var: ContextVar[Optional[str]] = ContextVar("trace_id", default=None)+_tenant_id_var: ContextVar[Optional[str]] = ContextVar("tenant_id", default=None)+++class LocalTraceQueue:+ def __init__(self, max_size: int = 10000):+ self._queue: asyncio.Queue[TraceEvent] = asyncio.Queue(maxsize=max_size)++ @property+ def size(self) -> int:+ return self._queue.qsize()++ def put_nowait(self, event: TraceEvent) -> bool:+ try:+ self._queue.put_nowait(event)+ return True+ except asyncio.QueueFull:+ return False++ async def get(self) -> TraceEvent:+ return await self._queue.get()++ def task_done(self) -> None:+ self._queue.task_done()++ def drain(self, limit: Optional[int] = None) -> list[TraceEvent]:+ events: list[TraceEvent] = []+ while not self._queue.empty() and (limit is None or len(events) < limit):+ events.append(self._queue.get_nowait())+ self._queue.task_done()+ return events+++class TraceTracker:+ def __init__(+ self,+ *,+ queue: Optional[LocalTraceQueue] = None,+ max_queue_size: Optional[int] = None,+ slow_operation_ms: Optional[float] = None,+ downtime_ms: Optional[float] = None,+ expensive_loop_iterations: Optional[int] = None,+ enabled: Optional[bool] = None,+ ):+ self.queue = queue or LocalTraceQueue(+ max_queue_size+ if max_queue_size is not None+ else int(os.getenv("TRACE_QUEUE_MAX_SIZE", "10000"))+ )+ self.slow_operation_ms = (+ slow_operation_ms+ if slow_operation_ms is not None+ else float(os.getenv("TRACE_SLOW_OPERATION_MS", "1000"))+ )+ self.downtime_ms = (+ downtime_ms+ if downtime_ms is not None+ else float(os.getenv("TRACE_DOWNTIME_MS", "5000"))+ )+ self.expensive_loop_iterations = (+ expensive_loop_iterations+ if expensive_loop_iterations is not None+ else int(os.getenv("TRACE_EXPENSIVE_LOOP_ITERATIONS", "10000"))+ )+ self.enabled = (+ enabled+ if enabled is not None+ else os.getenv("TRACE_TRACKER_ENABLED", "true") == "true"+ )+ self._subscribers: list[TraceSubscriber] = []+ self._subscriber_tasks: list[asyncio.Task] = []++ @property+ def default_sla_ms(self) -> float:+ return self.slow_operation_ms++ def set_context(+ self, *, trace_id: Optional[str] = None, tenant_id: Optional[str] = None+ ):+ trace_token = _trace_id_var.set(trace_id) if trace_id is not None else None+ tenant_token = _tenant_id_var.set(tenant_id) if tenant_id is not None else None+ return trace_token, tenant_token++ def reset_context(self, tokens) -> None:+ trace_token, tenant_token = tokens+ if trace_token is not None:+ _trace_id_var.reset(trace_token)+ if tenant_token is not None:+ _tenant_id_var.reset(tenant_token)++ def subscribe(self, subscriber: TraceSubscriber) -> None:+ self._subscribers.append(subscriber)++ def clear_subscribers(self) -> None:+ self._subscribers.clear()++ def publish(+ self,+ event_type: TraceEventType | str,+ name: str,+ *,+ severity: TraceSeverity | str = TraceSeverity.INFO,+ tenant_id: Optional[str] = None,+ trace_id: Optional[str] = None,+ service: Optional[str] = None,+ operation: Optional[str] = None,+ duration_ms: Optional[float] = None,+ threshold_ms: Optional[float] = None,+ status_code: Optional[int] = None,+ message: Optional[str] = None,+ metadata: Optional[dict[str, Any]] = None,+ exception: Optional[BaseException] = None,+ ) -> Optional[TraceEvent]:+ if not self.enabled:+ return None+ event_trace_id = trace_id or _trace_id_var.get()+ event_message = message or (str(exception) if exception else None)+ event = TraceEvent(+ event_type=TraceEventType(event_type),+ severity=TraceSeverity(severity),+ name=name,+ service=service,+ operation=operation,+ tenant_id=tenant_id or _tenant_id_var.get(),+ trace_id=event_trace_id,+ duration_ms=duration_ms,+ threshold_ms=threshold_ms,+ status_code=status_code,+ error_type=type(exception).__name__ if exception else None,+ message=event_message,+ metadata=dict(metadata or {}),+ stack_trace=(+ "".join(traceback.format_exception(exception)) if exception else None+ ),+ )+ if not self.queue.put_nowait(event):+ logger.warning("Trace queue is full; dropping event %s", event.name)+ return None+ return event++ def exception(+ self,+ name: str,+ exception: BaseException,+ *,+ service: Optional[str] = None,+ operation: Optional[str] = None,+ tenant_id: Optional[str] = None,+ trace_id: Optional[str] = None,+ duration_ms: Optional[float] = None,+ threshold_ms: Optional[float] = None,+ status_code: Optional[int] = None,+ metadata: Optional[dict[str, Any]] = None,+ ) -> Optional[TraceEvent]:+ return self.publish(+ TraceEventType.EXCEPTION,+ name,+ severity=TraceSeverity.ERROR,+ service=service,+ operation=operation,+ tenant_id=tenant_id,+ trace_id=trace_id,+ duration_ms=duration_ms,+ threshold_ms=threshold_ms,+ status_code=status_code,+ metadata=metadata,+ exception=exception,+ )++ def error(+ self,+ name: str,+ *,+ service: Optional[str] = None,+ operation: Optional[str] = None,+ tenant_id: Optional[str] = None,+ trace_id: Optional[str] = None,+ severity: TraceSeverity | str = TraceSeverity.ERROR,+ duration_ms: Optional[float] = None,+ threshold_ms: Optional[float] = None,+ status_code: Optional[int] = None,+ message: Optional[str] = None,+ metadata: Optional[dict[str, Any]] = None,+ ) -> Optional[TraceEvent]:+ return self.publish(+ TraceEventType.ERROR,+ name,+ severity=severity,+ service=service,+ operation=operation,+ tenant_id=tenant_id,+ trace_id=trace_id,+ duration_ms=duration_ms,+ threshold_ms=threshold_ms,+ status_code=status_code,+ message=message,+ metadata=metadata,+ )++ def downtime(+ self,+ name: str,+ duration_ms: float,+ *,+ service: Optional[str] = None,+ operation: Optional[str] = None,+ tenant_id: Optional[str] = None,+ trace_id: Optional[str] = None,+ metadata: Optional[dict[str, Any]] = None,+ ) -> Optional[TraceEvent]:+ return self.publish(+ TraceEventType.DOWNTIME,+ name,+ severity=TraceSeverity.CRITICAL,+ service=service,+ operation=operation,+ tenant_id=tenant_id,+ trace_id=trace_id,+ duration_ms=duration_ms,+ metadata=metadata,+ )++ def expensive_loop(+ self,+ name: str,+ iterations: int,+ *,+ service: Optional[str] = None,+ operation: Optional[str] = None,+ tenant_id: Optional[str] = None,+ trace_id: Optional[str] = None,+ metadata: Optional[dict[str, Any]] = None,+ ) -> Optional[TraceEvent]:+ if iterations < self.expensive_loop_iterations:+ return None+ return self.publish(+ TraceEventType.EXPENSIVE_LOOP,+ name,+ severity=TraceSeverity.WARNING,+ service=service,+ operation=operation,+ tenant_id=tenant_id,+ trace_id=trace_id,+ metadata={**(metadata or {}), "iterations": iterations},+ )++ @contextmanager+ def span(+ self,+ name: str,+ *,+ tenant_id: Optional[str] = None,+ trace_id: Optional[str] = None,+ metadata: Optional[dict[str, Any]] = None,+ slow_operation_ms: Optional[float] = None,+ ) -> Iterator[None]:+ tokens = self.set_context(trace_id=trace_id, tenant_id=tenant_id)+ started_at = time.perf_counter()+ try:+ yield+ except Exception as exc:+ duration_ms = (time.perf_counter() - started_at) * 1000+ self.exception(+ name,+ exc,+ metadata={**(metadata or {}), "duration_ms": duration_ms},+ )+ raise+ finally:+ duration_ms = (time.perf_counter() - started_at) * 1000+ threshold = (+ slow_operation_ms+ if slow_operation_ms is not None+ else self.slow_operation_ms+ )+ if duration_ms >= threshold:+ self.publish(+ TraceEventType.SLA_BREACH,+ name,+ severity=TraceSeverity.WARNING,+ duration_ms=duration_ms,+ threshold_ms=threshold,+ metadata=metadata,+ )+ self.reset_context(tokens)++ def track_loop(+ self,+ name: str,+ iterable,+ *,+ tenant_id: Optional[str] = None,+ trace_id: Optional[str] = None,+ metadata: Optional[dict[str, Any]] = None,+ threshold: Optional[int] = None,+ ):+ count = 0+ for item in iterable:+ count += 1+ yield item+ if count >= (threshold or self.expensive_loop_iterations):+ self.expensive_loop(+ name,+ count,+ tenant_id=tenant_id,+ trace_id=trace_id,+ metadata=metadata,+ )++ async def dispatch_once(self) -> TraceEvent:+ event = await self.queue.get()+ try:+ for subscriber in list(self._subscribers):+ result = subscriber.handle(event)+ if inspect.isawaitable(result):+ await result+ finally:+ self.queue.task_done()+ return event++ async def subscribe_forever(self) -> None:+ while True:+ await self.dispatch_once()++ def start_subscribers(self) -> None:+ if not self._subscribers or self._subscriber_tasks:+ return+ try:+ loop = asyncio.get_running_loop()+ except RuntimeError:+ return+ self._subscriber_tasks = [+ loop.create_task(self.subscribe_forever()) for _ in self._subscribers+ ]++ async def stop_subscribers(self) -> None:+ for task in self._subscriber_tasks:+ task.cancel()+ for task in self._subscriber_tasks:+ try:+ await task+ except asyncio.CancelledError:+ pass+ self._subscriber_tasks = []+++trace_tracker = TraceTracker()+tracer = trace_trackerdiff --git a/src/services/api/app.py b/src/services/api/app.pyindex cd50784..82adefa 100644--- a/src/services/api/app.py+++ b/src/services/api/app.py@@ -20,6 +20,7 @@ from src.services.api.routes.model import model_routerfrom src.services.api.routes.retrieve import retrieve_routerfrom src.services.api.routes.system import system_routerfrom src.services.api.routes.tasks import tasks_router+from src.lib.tracing.middleware import TraceMiddlewarelogger = logging.getLogger("brainapi.plugins")@@ -109,6 +110,7 @@ app = FastAPI(redirect_slashes=False,)+app.add_middleware(TraceMiddleware, service_name="brainapi-api")app.add_middleware(BrainPATMiddleware)app.add_middleware(BrainMiddleware)app.add_middleware(diff --git a/src/services/mcp/app.py b/src/services/mcp/app.pyindex 01f19e2..5bccb0f 100644--- a/src/services/mcp/app.py+++ b/src/services/mcp/app.py@@ -11,6 +11,7 @@ from starlette.routing import Route_project_root = Path(__file__).resolve().parent.parent.parent.parentdotenv.load_dotenv(_project_root / ".env")+from src.lib.tracing.middleware import TraceMiddlewarefrom src.services.mcp.main import auth_token_var, mcpPLUGINS_DIR = Path(os.getenv("PLUGINS_DIR", str(_project_root / "plugins")))@@ -118,6 +119,9 @@ _custom_routes = []app = Starlette(routes=_custom_routes + list(_mcp_app.routes),- middleware=[Middleware(AuthContextMiddleware)],+ middleware=[+ Middleware(AuthContextMiddleware),+ Middleware(TraceMiddleware, service_name="brainapi-mcp"),+ ],lifespan=_mcp_app.router.lifespan_context,)diff --git a/tests/test_tracing_sla_tracker.py b/tests/test_tracing_sla_tracker.pynew file mode 100644index 0000000..ed47b2a--- /dev/null+++ b/tests/test_tracing_sla_tracker.py@@ -0,0 +1,164 @@+import asyncio+import unittest++from starlette.applications import Starlette+from starlette.responses import JSONResponse, PlainTextResponse+from starlette.middleware import Middleware+from starlette.routing import Route+from starlette.testclient import TestClient++from src.lib.tracing import (+ LocalTraceQueue,+ TraceEvent,+ TraceEventType,+ TraceSeverity,+ TraceSubscriber,+ TraceTracker,+)+from src.lib.tracing.middleware import TraceMiddleware+from src.lib.tracing import middleware as tracing_middleware+++class MemorySubscriber(TraceSubscriber):+ def __init__(self):+ self.events = []++ def handle(self, event: TraceEvent) -> None:+ self.events.append(event)+++class TraceTrackerTests(unittest.TestCase):+ def test_publish_records_event_with_tenant_and_trace_context(self):+ tracker = TraceTracker(queue=LocalTraceQueue(), enabled=True)+ tokens = tracker.set_context(trace_id="trace-1", tenant_id="tenant-1")+ try:+ event = tracker.error(+ "operation.failed",+ service="api",+ operation="GET /demo",+ message="failed",+ )+ finally:+ tracker.reset_context(tokens)++ self.assertIsNotNone(event)+ drained = tracker.queue.drain()+ self.assertEqual(len(drained), 1)+ self.assertEqual(drained[0].event_type, TraceEventType.ERROR)+ self.assertEqual(drained[0].severity, TraceSeverity.ERROR)+ self.assertEqual(drained[0].tenant_id, "tenant-1")+ self.assertEqual(drained[0].trace_id, "trace-1")+ self.assertEqual(drained[0].service, "api")+ self.assertEqual(drained[0].operation, "GET /demo")++ def test_span_records_exception_and_sla_breach(self):+ tracker = TraceTracker(+ queue=LocalTraceQueue(),+ enabled=True,+ slow_operation_ms=0,+ )++ with self.assertRaises(ValueError):+ with tracker.span("failing-span", tenant_id="brain1"):+ raise ValueError("boom")++ events = tracker.queue.drain()+ self.assertEqual([event.event_type for event in events], [+ TraceEventType.EXCEPTION,+ TraceEventType.SLA_BREACH,+ ])+ self.assertEqual(events[0].error_type, "ValueError")+ self.assertIn("boom", events[0].message)+ self.assertEqual(events[0].tenant_id, "brain1")+ self.assertIsNotNone(events[0].stack_trace)++ def test_track_loop_records_expensive_loop_when_threshold_is_met(self):+ tracker = TraceTracker(+ queue=LocalTraceQueue(),+ enabled=True,+ expensive_loop_iterations=3,+ )++ self.assertEqual(list(tracker.track_loop("loop", range(3))), [0, 1, 2])++ events = tracker.queue.drain()+ self.assertEqual(len(events), 1)+ self.assertEqual(events[0].event_type, TraceEventType.EXPENSIVE_LOOP)+ self.assertEqual(events[0].metadata["iterations"], 3)++ def test_dispatch_once_sends_event_to_subscriber(self):+ async def run_test():+ tracker = TraceTracker(queue=LocalTraceQueue(), enabled=True)+ subscriber = MemorySubscriber()+ tracker.subscribe(subscriber)+ event = tracker.error("error")++ dispatched = await tracker.dispatch_once()++ self.assertIs(dispatched, event)+ self.assertEqual(subscriber.events, [event])++ asyncio.run(run_test())+++class TraceMiddlewareTests(unittest.TestCase):+ def setUp(self):+ self.original_tracer = tracing_middleware.tracer+ self.tracker = TraceTracker(queue=LocalTraceQueue(), enabled=True)+ tracing_middleware.tracer = self.tracker++ def tearDown(self):+ tracing_middleware.tracer = self.original_tracer++ def test_middleware_records_server_errors_and_tenant_id(self):+ async def error_route(_request):+ return JSONResponse({"detail": "bad"}, status_code=503)++ app = Starlette(+ routes=[Route("/bad", error_route)],+ middleware=[+ Middleware(+ TraceMiddleware,+ service_name="test-api",+ slow_request_ms=100000,+ )+ ],+ )++ response = TestClient(app).get("/bad", headers={"X-Brain-ID": "tenant-a"})++ self.assertEqual(response.status_code, 503)+ events = self.tracker.queue.drain()+ self.assertEqual(len(events), 1)+ self.assertEqual(events[0].event_type, TraceEventType.ERROR)+ self.assertEqual(events[0].tenant_id, "tenant-a")+ self.assertEqual(events[0].service, "test-api")+ self.assertEqual(events[0].status_code, 503)++ def test_middleware_records_sla_breach(self):+ async def ok_route(_request):+ return PlainTextResponse("ok")++ app = Starlette(+ routes=[Route("/ok", ok_route)],+ middleware=[+ Middleware(+ TraceMiddleware,+ service_name="test-api",+ slow_request_ms=0,+ )+ ],+ )++ response = TestClient(app).get("/ok?brain_id=tenant-b")++ self.assertEqual(response.status_code, 200)+ events = self.tracker.queue.drain()+ self.assertEqual(len(events), 1)+ self.assertEqual(events[0].event_type, TraceEventType.SLA_BREACH)+ self.assertEqual(events[0].tenant_id, "tenant-b")+ self.assertEqual(events[0].operation, "GET /ok")+++if __name__ == "__main__":+ unittest.main()