BrainAPI is a knowledge graph–powered AI memory layer that transforms unstructured data into structured knowledge, enabling intelligent search, recommendations, and contextual memory for AI agents and applications.
Python ai-agentsai-memoryknowledge-graphknowledge-managementknowledge-representationreccomendation-system
Add runtime health and resource tracing
Cursor Agent714b59d3ca47
diff --git a/src/lib/tracing/__init__.py b/src/lib/tracing/__init__.pyindex 6ee1544..33ee078 100644--- a/src/lib/tracing/__init__.py+++ b/src/lib/tracing/__init__.py@@ -3,16 +3,26 @@ from src.lib.tracing.events import (TraceEventType,TraceSeverity,)+from src.lib.tracing.runtime import (+ HealthProbe,+ RuntimeMonitor,+ start_runtime_monitoring,+ stop_runtime_monitoring,+)from src.lib.tracing.subscribers import TraceSubscriberfrom src.lib.tracing.tracker import LocalTraceQueue, TraceTracker, trace_tracker, tracer__all__ = [+ "HealthProbe","LocalTraceQueue",+ "RuntimeMonitor","TraceEvent","TraceEventType","TraceSeverity","TraceSubscriber","TraceTracker",+ "start_runtime_monitoring",+ "stop_runtime_monitoring","trace_tracker","tracer",]diff --git a/src/lib/tracing/events.py b/src/lib/tracing/events.pyindex 21e9410..bd8ffca 100644--- a/src/lib/tracing/events.py+++ b/src/lib/tracing/events.py@@ -11,6 +11,10 @@ class TraceEventType(str, Enum):EXPENSIVE_LOOP = "expensive_loop"SLA_BREACH = "sla_breach"LATENCY = "latency"+ HEARTBEAT = "heartbeat"+ HEALTH_CHECK = "health_check"+ RESOURCE_SAMPLE = "resource_sample"+ PROCESS = "process"class TraceSeverity(str, Enum):diff --git a/src/lib/tracing/runtime.py b/src/lib/tracing/runtime.pynew file mode 100644index 0000000..893148a--- /dev/null+++ b/src/lib/tracing/runtime.py@@ -0,0 +1,336 @@+import atexit+import asyncio+import os+import resource+import socket+import sys+import threading+import time+import traceback+from contextlib import asynccontextmanager+from dataclasses import dataclass+from typing import Callable++from src.lib.tracing.events import TraceEventType, TraceSeverity+from src.lib.tracing.tracker import tracer+++@dataclass(frozen=True)+class HealthProbe:+ name: str+ host: str+ port: int+ timeout_seconds: float = 1.0+++class RuntimeMonitor:+ def __init__(+ self,+ *,+ service_name: str,+ heartbeat_interval_seconds: float | None = None,+ health_interval_seconds: float | None = None,+ resource_interval_seconds: float | None = None,+ probes: list[HealthProbe] | None = None,+ resource_sampler: Callable[[], dict] | None = None,+ ):+ self.service_name = service_name+ self.heartbeat_interval_seconds = _interval(+ heartbeat_interval_seconds, "TRACE_HEARTBEAT_INTERVAL_SECONDS", 30+ )+ self.health_interval_seconds = _interval(+ health_interval_seconds, "TRACE_HEALTH_INTERVAL_SECONDS", 30+ )+ self.resource_interval_seconds = _interval(+ resource_interval_seconds, "TRACE_RESOURCE_INTERVAL_SECONDS", 30+ )+ self.probes = probes or default_health_probes()+ self.resource_sampler = resource_sampler or sample_process_resources+ self._stop_event = threading.Event()+ self._thread: threading.Thread | None = None+ self._hooks_installed = False++ def start(self) -> None:+ self.install_exception_hooks()+ tracer.publish(+ TraceEventType.PROCESS,+ "process.started",+ service=self.service_name,+ operation="runtime.start",+ metadata=_process_metadata(),+ )+ if self._thread and self._thread.is_alive():+ return+ self._thread = threading.Thread(+ target=self._run,+ name=f"{self.service_name}-runtime-monitor",+ daemon=True,+ )+ self._thread.start()++ def stop(self) -> None:+ self._stop_event.set()+ if self._thread and self._thread.is_alive():+ self._thread.join(timeout=2)+ tracer.publish(+ TraceEventType.PROCESS,+ "process.stopped",+ service=self.service_name,+ operation="runtime.stop",+ metadata=_process_metadata(),+ )++ def install_exception_hooks(self) -> None:+ if self._hooks_installed:+ return+ previous_sys_hook = sys.excepthook+ previous_thread_hook = getattr(threading, "excepthook", None)++ def sys_hook(exc_type, exc_value, exc_traceback):+ self.record_unhandled_exception(exc_value, exc_traceback, "sys.excepthook")+ previous_sys_hook(exc_type, exc_value, exc_traceback)++ def thread_hook(args):+ self.record_unhandled_exception(+ args.exc_value,+ args.exc_traceback,+ f"threading.excepthook:{getattr(args.thread, 'name', 'unknown')}",+ )+ if previous_thread_hook:+ previous_thread_hook(args)++ sys.excepthook = sys_hook+ if previous_thread_hook:+ threading.excepthook = thread_hook+ self._hooks_installed = True++ def install_asyncio_exception_handler(self, loop: asyncio.AbstractEventLoop) -> None:+ previous_handler = loop.get_exception_handler()++ def handler(event_loop, context):+ exc = context.get("exception")+ if exc:+ self.record_unhandled_exception(exc, exc.__traceback__, "asyncio")+ else:+ tracer.error(+ "runtime.asyncio.unhandled_error",+ service=self.service_name,+ operation="asyncio",+ severity=TraceSeverity.ERROR,+ message=context.get("message"),+ metadata={k: str(v) for k, v in context.items() if k != "exception"},+ )+ if previous_handler:+ previous_handler(event_loop, context)+ else:+ event_loop.default_exception_handler(context)++ loop.set_exception_handler(handler)++ def record_unhandled_exception(+ self,+ exc: BaseException,+ exc_traceback,+ operation: str,+ ) -> None:+ tracer.publish(+ TraceEventType.EXCEPTION,+ "runtime.unhandled_exception",+ service=self.service_name,+ operation=operation,+ severity=TraceSeverity.CRITICAL,+ message=str(exc),+ exception=exc,+ metadata={+ **_process_metadata(),+ "traceback": "".join(+ traceback.format_exception(type(exc), exc, exc_traceback)+ ),+ },+ )++ def heartbeat(self) -> None:+ tracer.publish(+ TraceEventType.HEARTBEAT,+ "runtime.heartbeat",+ service=self.service_name,+ operation="heartbeat",+ metadata=_process_metadata(),+ )++ def sample_resources(self) -> None:+ tracer.publish(+ TraceEventType.RESOURCE_SAMPLE,+ "runtime.resources",+ service=self.service_name,+ operation="resource_sampler",+ metadata=self.resource_sampler(),+ )++ def check_health(self) -> None:+ for probe in self.probes:+ started_at = time.perf_counter()+ ok = probe_tcp(probe)+ duration_ms = (time.perf_counter() - started_at) * 1000+ metadata = {+ "probe": probe.name,+ "host": probe.host,+ "port": probe.port,+ "ok": ok,+ }+ if ok:+ tracer.publish(+ TraceEventType.HEALTH_CHECK,+ f"health.{probe.name}",+ service=self.service_name,+ operation="health_check",+ duration_ms=duration_ms,+ metadata=metadata,+ )+ else:+ tracer.downtime(+ f"health.{probe.name}.down",+ duration_ms,+ service=self.service_name,+ operation="health_check",+ metadata=metadata,+ )++ def _run(self) -> None:+ last_heartbeat = last_health = last_resource = 0.0+ while not self._stop_event.is_set():+ now = time.monotonic()+ if _due(now, last_heartbeat, self.heartbeat_interval_seconds):+ self.heartbeat()+ last_heartbeat = now+ if _due(now, last_health, self.health_interval_seconds):+ self.check_health()+ last_health = now+ if _due(now, last_resource, self.resource_interval_seconds):+ self.sample_resources()+ last_resource = now+ self._stop_event.wait(1)+++_monitors: dict[str, RuntimeMonitor] = {}+++def start_runtime_monitoring(service_name: str, **kwargs) -> RuntimeMonitor:+ monitor = _monitors.get(service_name)+ if monitor is None:+ monitor = RuntimeMonitor(service_name=service_name, **kwargs)+ _monitors[service_name] = monitor+ atexit.register(monitor.stop)+ monitor.start()+ return monitor+++def stop_runtime_monitoring(service_name: str) -> None:+ monitor = _monitors.get(service_name)+ if monitor:+ monitor.stop()+++def runtime_tracing_lifespan(service_name: str, nested_lifespan=None):+ @asynccontextmanager+ async def lifespan(app):+ start_runtime_monitoring(service_name)+ if nested_lifespan is None:+ try:+ yield+ finally:+ stop_runtime_monitoring(service_name)+ return+ async with nested_lifespan(app):+ try:+ yield+ finally:+ stop_runtime_monitoring(service_name)++ return lifespan+++def default_health_probes() -> list[HealthProbe]:+ return [+ _probe_from_env("redis", "REDIS_HOST", "REDIS_PORT"),+ _probe_from_env("mongo", "MONGO_HOST", "MONGO_PORT"),+ _probe_from_env("neo4j", "NEO4J_HOST", "NEO4J_PORT"),+ _probe_from_env("milvus", "MILVUS_HOST", "MILVUS_PORT"),+ _probe_from_env("rabbitmq", "RABBITMQ_HOST", "RABBITMQ_PORT"),+ ]+++def _probe_from_env(name: str, host_var: str, port_var: str) -> HealthProbe:+ defaults = {+ "redis": ("localhost", 6379),+ "mongo": ("localhost", 27017),+ "neo4j": ("localhost", 7687),+ "milvus": ("localhost", 19530),+ "rabbitmq": ("localhost", 5672),+ }+ default_host, default_port = defaults[name]+ return HealthProbe(+ name=name,+ host=os.getenv(host_var, default_host),+ port=int(os.getenv(port_var, str(default_port))),+ timeout_seconds=float(os.getenv("TRACE_HEALTH_TIMEOUT_SECONDS", "1")),+ )+++def probe_tcp(probe: HealthProbe) -> bool:+ try:+ with socket.create_connection(+ (probe.host, probe.port), timeout=probe.timeout_seconds+ ):+ return True+ except OSError:+ return False+++def sample_process_resources() -> dict:+ usage = resource.getrusage(resource.RUSAGE_SELF)+ return {+ **_process_metadata(),+ "rss_kb": usage.ru_maxrss,+ "user_cpu_seconds": usage.ru_utime,+ "system_cpu_seconds": usage.ru_stime,+ "threads": threading.active_count(),+ "load_avg": os.getloadavg() if hasattr(os, "getloadavg") else None,+ **_proc_status(),+ }+++def _proc_status() -> dict:+ status_path = "/proc/self/status"+ if not os.path.exists(status_path):+ return {}+ keys = {+ "VmRSS": "vm_rss",+ "VmSize": "vm_size",+ "Threads": "proc_threads",+ }+ values = {}+ try:+ with open(status_path, encoding="utf-8") as status_file:+ for line in status_file:+ key, _, value = line.partition(":")+ if key in keys:+ values[keys[key]] = value.strip()+ except OSError:+ return {}+ return values+++def _process_metadata() -> dict:+ return {+ "pid": os.getpid(),+ "process": os.path.basename(sys.argv[0] or "python"),+ }+++def _interval(value: float | None, env_name: str, default: float) -> float:+ return value if value is not None else float(os.getenv(env_name, str(default)))+++def _due(now: float, last: float, interval: float) -> bool:+ return interval >= 0 and now - last >= intervaldiff --git a/src/services/api/app.py b/src/services/api/app.pyindex 42026b7..f658cfb 100644--- a/src/services/api/app.py+++ b/src/services/api/app.py@@ -21,6 +21,7 @@ from src.services.api.routes.retrieve import retrieve_routerfrom src.services.api.routes.system import system_routerfrom src.services.api.routes.tasks import tasks_routerfrom src.lib.tracing.middleware import TraceMiddleware+from src.lib.tracing.runtime import start_runtime_monitoring, stop_runtime_monitoringlogger = logging.getLogger("brainapi.plugins")@@ -32,6 +33,7 @@ async def lifespan(app: FastAPI):from src.core.plugins.context import PluginContextfrom src.core.plugins.loader import PluginLoader+ start_runtime_monitoring("brainapi-api")ctx = PluginContext.from_app(app)loader = PluginLoader(plugins_dir=PLUGINS_DIR, context=ctx)results = loader.load_all()@@ -43,7 +45,10 @@ async def lifespan(app: FastAPI):for handler in handlers:await handler() if _is_coroutine(handler) else handler()- yield+ try:+ yield+ finally:+ stop_runtime_monitoring("brainapi-api")for event_name, handlers in ctx._event_handlers.items():if event_name == "shutdown":diff --git a/src/services/mcp/app.py b/src/services/mcp/app.pyindex 92ddabe..fd01163 100644--- a/src/services/mcp/app.py+++ b/src/services/mcp/app.py@@ -11,7 +11,10 @@ from starlette.routing import Route_project_root = Path(__file__).resolve().parent.parent.parent.parentdotenv.load_dotenv(_project_root / ".env")+from contextlib import asynccontextmanager+from src.lib.tracing.middleware import TraceMiddleware+from src.lib.tracing.runtime import start_runtime_monitoring, stop_runtime_monitoringfrom src.services.mcp.main import auth_token_var, mcpPLUGINS_DIR = Path(os.getenv("PLUGINS_DIR", str(_project_root / "plugins")))@@ -80,6 +83,16 @@ _load_mcp_plugins()_mcp_app = mcp.streamable_http_app()+@asynccontextmanager+async def _lifespan(app):+ start_runtime_monitoring("brainapi-mcp")+ async with _mcp_app.router.lifespan_context(app):+ try:+ yield+ finally:+ stop_runtime_monitoring("brainapi-mcp")++class AuthContextMiddleware:def __init__(self, app):self.app = app@@ -123,5 +136,5 @@ app = Starlette(Middleware(TraceMiddleware, service_name="brainapi-mcp"),Middleware(AuthContextMiddleware),],- lifespan=_mcp_app.router.lifespan_context,+ lifespan=_lifespan,)diff --git a/src/workers/app.py b/src/workers/app.pyindex aa4e75b..e5896e2 100644--- a/src/workers/app.py+++ b/src/workers/app.py@@ -31,6 +31,7 @@ from celery import Celeryfrom kombu import Queuefrom src.config import config+from src.lib.tracing import start_runtime_monitoringfrom src.lib.tracing.workers import register_celery_tracingos.environ.setdefault("GRPC_ENABLE_FORK_SUPPORT", "1")@@ -100,3 +101,4 @@ ingestion_app.conf.update(},)register_celery_tracing(service_name="brainapi-worker")+start_runtime_monitoring(service_name="brainapi-worker")diff --git a/tests/test_tracing_runtime_monitor.py b/tests/test_tracing_runtime_monitor.pynew file mode 100644index 0000000..6934c2f--- /dev/null+++ b/tests/test_tracing_runtime_monitor.py@@ -0,0 +1,88 @@+import asyncio+import unittest+from unittest.mock import patch++from src.lib.tracing import LocalTraceQueue, TraceEventType, TraceTracker+from src.lib.tracing import runtime as tracing_runtime+from src.lib.tracing.runtime import HealthProbe, RuntimeMonitor+++class RuntimeMonitorTests(unittest.TestCase):+ def setUp(self):+ self.original_tracer = tracing_runtime.tracer+ self.tracker = TraceTracker(queue=LocalTraceQueue(), enabled=True)+ tracing_runtime.tracer = self.tracker++ def tearDown(self):+ tracing_runtime.tracer = self.original_tracer+ tracing_runtime._monitors.clear()++ def test_runtime_monitor_records_heartbeat_resources_and_shutdown(self):+ monitor = RuntimeMonitor(+ service_name="test-service",+ probes=[],+ resource_sampler=lambda: {"rss_kb": 12},+ )++ monitor.heartbeat()+ monitor.sample_resources()+ monitor.stop()++ events = self.tracker.queue.drain()+ self.assertEqual(events[0].event_type, TraceEventType.HEARTBEAT)+ self.assertEqual(events[1].event_type, TraceEventType.RESOURCE_SAMPLE)+ self.assertEqual(events[1].metadata["rss_kb"], 12)+ self.assertEqual(events[2].event_type, TraceEventType.PROCESS)+ self.assertEqual(events[2].name, "process.stopped")++ def test_health_check_records_success_and_downtime(self):+ monitor = RuntimeMonitor(+ service_name="test-service",+ probes=[+ HealthProbe("up", "127.0.0.1", 1),+ HealthProbe("down", "127.0.0.1", 2),+ ],+ )++ with patch.object(tracing_runtime, "probe_tcp", side_effect=[True, False]):+ monitor.check_health()++ events = self.tracker.queue.drain()+ self.assertEqual(events[0].event_type, TraceEventType.HEALTH_CHECK)+ self.assertEqual(events[0].name, "health.up")+ self.assertEqual(events[1].event_type, TraceEventType.DOWNTIME)+ self.assertEqual(events[1].name, "health.down.down")++ def test_unhandled_exception_hook_records_exception(self):+ monitor = RuntimeMonitor(service_name="test-service", probes=[])+ exc = RuntimeError("boom")++ monitor.record_unhandled_exception(exc, exc.__traceback__, "unit-test")++ events = self.tracker.queue.drain()+ self.assertEqual(len(events), 1)+ self.assertEqual(events[0].event_type, TraceEventType.EXCEPTION)+ self.assertEqual(events[0].severity.value, "critical")+ self.assertEqual(events[0].operation, "unit-test")++ def test_asyncio_exception_handler_records_context(self):+ async def run_test():+ loop = asyncio.get_running_loop()+ previous_handler = loop.get_exception_handler()+ loop.set_exception_handler(lambda _loop, _context: None)+ monitor = RuntimeMonitor(service_name="test-service", probes=[])+ monitor.install_asyncio_exception_handler(loop)+ try:+ loop.call_exception_handler({"message": "async problem"})+ finally:+ loop.set_exception_handler(previous_handler)++ asyncio.run(run_test())++ events = self.tracker.queue.drain()+ self.assertEqual(events[0].event_type, TraceEventType.ERROR)+ self.assertEqual(events[0].name, "runtime.asyncio.unhandled_error")+++if __name__ == "__main__":+ unittest.main()