Skip to content

BrainAPI is a knowledge graph–powered AI memory layer that transforms unstructured data into structured knowledge, enabling intelligent search, recommendations, and contextual memory for AI agents and applications.

Add runtime health and resource tracing

Cursor Agent714b59d3ca47
diff --git a/src/lib/tracing/__init__.py b/src/lib/tracing/__init__.py
index 6ee1544..33ee078 100644
--- a/src/lib/tracing/__init__.py
+++ b/src/lib/tracing/__init__.py
@@ -3,16 +3,26 @@ from src.lib.tracing.events import (
TraceEventType,
TraceSeverity,
)
+from src.lib.tracing.runtime import (
+ HealthProbe,
+ RuntimeMonitor,
+ start_runtime_monitoring,
+ stop_runtime_monitoring,
+)
from src.lib.tracing.subscribers import TraceSubscriber
from src.lib.tracing.tracker import LocalTraceQueue, TraceTracker, trace_tracker, tracer
__all__ = [
+ "HealthProbe",
"LocalTraceQueue",
+ "RuntimeMonitor",
"TraceEvent",
"TraceEventType",
"TraceSeverity",
"TraceSubscriber",
"TraceTracker",
+ "start_runtime_monitoring",
+ "stop_runtime_monitoring",
"trace_tracker",
"tracer",
]
diff --git a/src/lib/tracing/events.py b/src/lib/tracing/events.py
index 21e9410..bd8ffca 100644
--- a/src/lib/tracing/events.py
+++ b/src/lib/tracing/events.py
@@ -11,6 +11,10 @@ class TraceEventType(str, Enum):
EXPENSIVE_LOOP = "expensive_loop"
SLA_BREACH = "sla_breach"
LATENCY = "latency"
+ HEARTBEAT = "heartbeat"
+ HEALTH_CHECK = "health_check"
+ RESOURCE_SAMPLE = "resource_sample"
+ PROCESS = "process"
class TraceSeverity(str, Enum):
diff --git a/src/lib/tracing/runtime.py b/src/lib/tracing/runtime.py
new file mode 100644
index 0000000..893148a
--- /dev/null
+++ b/src/lib/tracing/runtime.py
@@ -0,0 +1,336 @@
+import atexit
+import asyncio
+import os
+import resource
+import socket
+import sys
+import threading
+import time
+import traceback
+from contextlib import asynccontextmanager
+from dataclasses import dataclass
+from typing import Callable
+
+from src.lib.tracing.events import TraceEventType, TraceSeverity
+from src.lib.tracing.tracker import tracer
+
+
+@dataclass(frozen=True)
+class HealthProbe:
+ name: str
+ host: str
+ port: int
+ timeout_seconds: float = 1.0
+
+
+class RuntimeMonitor:
+ def __init__(
+ self,
+ *,
+ service_name: str,
+ heartbeat_interval_seconds: float | None = None,
+ health_interval_seconds: float | None = None,
+ resource_interval_seconds: float | None = None,
+ probes: list[HealthProbe] | None = None,
+ resource_sampler: Callable[[], dict] | None = None,
+ ):
+ self.service_name = service_name
+ self.heartbeat_interval_seconds = _interval(
+ heartbeat_interval_seconds, "TRACE_HEARTBEAT_INTERVAL_SECONDS", 30
+ )
+ self.health_interval_seconds = _interval(
+ health_interval_seconds, "TRACE_HEALTH_INTERVAL_SECONDS", 30
+ )
+ self.resource_interval_seconds = _interval(
+ resource_interval_seconds, "TRACE_RESOURCE_INTERVAL_SECONDS", 30
+ )
+ self.probes = probes or default_health_probes()
+ self.resource_sampler = resource_sampler or sample_process_resources
+ self._stop_event = threading.Event()
+ self._thread: threading.Thread | None = None
+ self._hooks_installed = False
+
+ def start(self) -> None:
+ self.install_exception_hooks()
+ tracer.publish(
+ TraceEventType.PROCESS,
+ "process.started",
+ service=self.service_name,
+ operation="runtime.start",
+ metadata=_process_metadata(),
+ )
+ if self._thread and self._thread.is_alive():
+ return
+ self._thread = threading.Thread(
+ target=self._run,
+ name=f"{self.service_name}-runtime-monitor",
+ daemon=True,
+ )
+ self._thread.start()
+
+ def stop(self) -> None:
+ self._stop_event.set()
+ if self._thread and self._thread.is_alive():
+ self._thread.join(timeout=2)
+ tracer.publish(
+ TraceEventType.PROCESS,
+ "process.stopped",
+ service=self.service_name,
+ operation="runtime.stop",
+ metadata=_process_metadata(),
+ )
+
+ def install_exception_hooks(self) -> None:
+ if self._hooks_installed:
+ return
+ previous_sys_hook = sys.excepthook
+ previous_thread_hook = getattr(threading, "excepthook", None)
+
+ def sys_hook(exc_type, exc_value, exc_traceback):
+ self.record_unhandled_exception(exc_value, exc_traceback, "sys.excepthook")
+ previous_sys_hook(exc_type, exc_value, exc_traceback)
+
+ def thread_hook(args):
+ self.record_unhandled_exception(
+ args.exc_value,
+ args.exc_traceback,
+ f"threading.excepthook:{getattr(args.thread, 'name', 'unknown')}",
+ )
+ if previous_thread_hook:
+ previous_thread_hook(args)
+
+ sys.excepthook = sys_hook
+ if previous_thread_hook:
+ threading.excepthook = thread_hook
+ self._hooks_installed = True
+
+ def install_asyncio_exception_handler(self, loop: asyncio.AbstractEventLoop) -> None:
+ previous_handler = loop.get_exception_handler()
+
+ def handler(event_loop, context):
+ exc = context.get("exception")
+ if exc:
+ self.record_unhandled_exception(exc, exc.__traceback__, "asyncio")
+ else:
+ tracer.error(
+ "runtime.asyncio.unhandled_error",
+ service=self.service_name,
+ operation="asyncio",
+ severity=TraceSeverity.ERROR,
+ message=context.get("message"),
+ metadata={k: str(v) for k, v in context.items() if k != "exception"},
+ )
+ if previous_handler:
+ previous_handler(event_loop, context)
+ else:
+ event_loop.default_exception_handler(context)
+
+ loop.set_exception_handler(handler)
+
+ def record_unhandled_exception(
+ self,
+ exc: BaseException,
+ exc_traceback,
+ operation: str,
+ ) -> None:
+ tracer.publish(
+ TraceEventType.EXCEPTION,
+ "runtime.unhandled_exception",
+ service=self.service_name,
+ operation=operation,
+ severity=TraceSeverity.CRITICAL,
+ message=str(exc),
+ exception=exc,
+ metadata={
+ **_process_metadata(),
+ "traceback": "".join(
+ traceback.format_exception(type(exc), exc, exc_traceback)
+ ),
+ },
+ )
+
+ def heartbeat(self) -> None:
+ tracer.publish(
+ TraceEventType.HEARTBEAT,
+ "runtime.heartbeat",
+ service=self.service_name,
+ operation="heartbeat",
+ metadata=_process_metadata(),
+ )
+
+ def sample_resources(self) -> None:
+ tracer.publish(
+ TraceEventType.RESOURCE_SAMPLE,
+ "runtime.resources",
+ service=self.service_name,
+ operation="resource_sampler",
+ metadata=self.resource_sampler(),
+ )
+
+ def check_health(self) -> None:
+ for probe in self.probes:
+ started_at = time.perf_counter()
+ ok = probe_tcp(probe)
+ duration_ms = (time.perf_counter() - started_at) * 1000
+ metadata = {
+ "probe": probe.name,
+ "host": probe.host,
+ "port": probe.port,
+ "ok": ok,
+ }
+ if ok:
+ tracer.publish(
+ TraceEventType.HEALTH_CHECK,
+ f"health.{probe.name}",
+ service=self.service_name,
+ operation="health_check",
+ duration_ms=duration_ms,
+ metadata=metadata,
+ )
+ else:
+ tracer.downtime(
+ f"health.{probe.name}.down",
+ duration_ms,
+ service=self.service_name,
+ operation="health_check",
+ metadata=metadata,
+ )
+
+ def _run(self) -> None:
+ last_heartbeat = last_health = last_resource = 0.0
+ while not self._stop_event.is_set():
+ now = time.monotonic()
+ if _due(now, last_heartbeat, self.heartbeat_interval_seconds):
+ self.heartbeat()
+ last_heartbeat = now
+ if _due(now, last_health, self.health_interval_seconds):
+ self.check_health()
+ last_health = now
+ if _due(now, last_resource, self.resource_interval_seconds):
+ self.sample_resources()
+ last_resource = now
+ self._stop_event.wait(1)
+
+
+_monitors: dict[str, RuntimeMonitor] = {}
+
+
+def start_runtime_monitoring(service_name: str, **kwargs) -> RuntimeMonitor:
+ monitor = _monitors.get(service_name)
+ if monitor is None:
+ monitor = RuntimeMonitor(service_name=service_name, **kwargs)
+ _monitors[service_name] = monitor
+ atexit.register(monitor.stop)
+ monitor.start()
+ return monitor
+
+
+def stop_runtime_monitoring(service_name: str) -> None:
+ monitor = _monitors.get(service_name)
+ if monitor:
+ monitor.stop()
+
+
+def runtime_tracing_lifespan(service_name: str, nested_lifespan=None):
+ @asynccontextmanager
+ async def lifespan(app):
+ start_runtime_monitoring(service_name)
+ if nested_lifespan is None:
+ try:
+ yield
+ finally:
+ stop_runtime_monitoring(service_name)
+ return
+ async with nested_lifespan(app):
+ try:
+ yield
+ finally:
+ stop_runtime_monitoring(service_name)
+
+ return lifespan
+
+
+def default_health_probes() -> list[HealthProbe]:
+ return [
+ _probe_from_env("redis", "REDIS_HOST", "REDIS_PORT"),
+ _probe_from_env("mongo", "MONGO_HOST", "MONGO_PORT"),
+ _probe_from_env("neo4j", "NEO4J_HOST", "NEO4J_PORT"),
+ _probe_from_env("milvus", "MILVUS_HOST", "MILVUS_PORT"),
+ _probe_from_env("rabbitmq", "RABBITMQ_HOST", "RABBITMQ_PORT"),
+ ]
+
+
+def _probe_from_env(name: str, host_var: str, port_var: str) -> HealthProbe:
+ defaults = {
+ "redis": ("localhost", 6379),
+ "mongo": ("localhost", 27017),
+ "neo4j": ("localhost", 7687),
+ "milvus": ("localhost", 19530),
+ "rabbitmq": ("localhost", 5672),
+ }
+ default_host, default_port = defaults[name]
+ return HealthProbe(
+ name=name,
+ host=os.getenv(host_var, default_host),
+ port=int(os.getenv(port_var, str(default_port))),
+ timeout_seconds=float(os.getenv("TRACE_HEALTH_TIMEOUT_SECONDS", "1")),
+ )
+
+
+def probe_tcp(probe: HealthProbe) -> bool:
+ try:
+ with socket.create_connection(
+ (probe.host, probe.port), timeout=probe.timeout_seconds
+ ):
+ return True
+ except OSError:
+ return False
+
+
+def sample_process_resources() -> dict:
+ usage = resource.getrusage(resource.RUSAGE_SELF)
+ return {
+ **_process_metadata(),
+ "rss_kb": usage.ru_maxrss,
+ "user_cpu_seconds": usage.ru_utime,
+ "system_cpu_seconds": usage.ru_stime,
+ "threads": threading.active_count(),
+ "load_avg": os.getloadavg() if hasattr(os, "getloadavg") else None,
+ **_proc_status(),
+ }
+
+
+def _proc_status() -> dict:
+ status_path = "/proc/self/status"
+ if not os.path.exists(status_path):
+ return {}
+ keys = {
+ "VmRSS": "vm_rss",
+ "VmSize": "vm_size",
+ "Threads": "proc_threads",
+ }
+ values = {}
+ try:
+ with open(status_path, encoding="utf-8") as status_file:
+ for line in status_file:
+ key, _, value = line.partition(":")
+ if key in keys:
+ values[keys[key]] = value.strip()
+ except OSError:
+ return {}
+ return values
+
+
+def _process_metadata() -> dict:
+ return {
+ "pid": os.getpid(),
+ "process": os.path.basename(sys.argv[0] or "python"),
+ }
+
+
+def _interval(value: float | None, env_name: str, default: float) -> float:
+ return value if value is not None else float(os.getenv(env_name, str(default)))
+
+
+def _due(now: float, last: float, interval: float) -> bool:
+ return interval >= 0 and now - last >= interval
diff --git a/src/services/api/app.py b/src/services/api/app.py
index 42026b7..f658cfb 100644
--- a/src/services/api/app.py
+++ b/src/services/api/app.py
@@ -21,6 +21,7 @@ from src.services.api.routes.retrieve import retrieve_router
from src.services.api.routes.system import system_router
from src.services.api.routes.tasks import tasks_router
from src.lib.tracing.middleware import TraceMiddleware
+from src.lib.tracing.runtime import start_runtime_monitoring, stop_runtime_monitoring
logger = logging.getLogger("brainapi.plugins")
@@ -32,6 +33,7 @@ async def lifespan(app: FastAPI):
from src.core.plugins.context import PluginContext
from src.core.plugins.loader import PluginLoader
+ start_runtime_monitoring("brainapi-api")
ctx = PluginContext.from_app(app)
loader = PluginLoader(plugins_dir=PLUGINS_DIR, context=ctx)
results = loader.load_all()
@@ -43,7 +45,10 @@ async def lifespan(app: FastAPI):
for handler in handlers:
await handler() if _is_coroutine(handler) else handler()
- yield
+ try:
+ yield
+ finally:
+ stop_runtime_monitoring("brainapi-api")
for event_name, handlers in ctx._event_handlers.items():
if event_name == "shutdown":
diff --git a/src/services/mcp/app.py b/src/services/mcp/app.py
index 92ddabe..fd01163 100644
--- a/src/services/mcp/app.py
+++ b/src/services/mcp/app.py
@@ -11,7 +11,10 @@ from starlette.routing import Route
_project_root = Path(__file__).resolve().parent.parent.parent.parent
dotenv.load_dotenv(_project_root / ".env")
+from contextlib import asynccontextmanager
+
from src.lib.tracing.middleware import TraceMiddleware
+from src.lib.tracing.runtime import start_runtime_monitoring, stop_runtime_monitoring
from src.services.mcp.main import auth_token_var, mcp
PLUGINS_DIR = Path(os.getenv("PLUGINS_DIR", str(_project_root / "plugins")))
@@ -80,6 +83,16 @@ _load_mcp_plugins()
_mcp_app = mcp.streamable_http_app()
+@asynccontextmanager
+async def _lifespan(app):
+ start_runtime_monitoring("brainapi-mcp")
+ async with _mcp_app.router.lifespan_context(app):
+ try:
+ yield
+ finally:
+ stop_runtime_monitoring("brainapi-mcp")
+
+
class AuthContextMiddleware:
def __init__(self, app):
self.app = app
@@ -123,5 +136,5 @@ app = Starlette(
Middleware(TraceMiddleware, service_name="brainapi-mcp"),
Middleware(AuthContextMiddleware),
],
- lifespan=_mcp_app.router.lifespan_context,
+ lifespan=_lifespan,
)
diff --git a/src/workers/app.py b/src/workers/app.py
index aa4e75b..e5896e2 100644
--- a/src/workers/app.py
+++ b/src/workers/app.py
@@ -31,6 +31,7 @@ from celery import Celery
from kombu import Queue
from src.config import config
+from src.lib.tracing import start_runtime_monitoring
from src.lib.tracing.workers import register_celery_tracing
os.environ.setdefault("GRPC_ENABLE_FORK_SUPPORT", "1")
@@ -100,3 +101,4 @@ ingestion_app.conf.update(
},
)
register_celery_tracing(service_name="brainapi-worker")
+start_runtime_monitoring(service_name="brainapi-worker")
diff --git a/tests/test_tracing_runtime_monitor.py b/tests/test_tracing_runtime_monitor.py
new file mode 100644
index 0000000..6934c2f
--- /dev/null
+++ b/tests/test_tracing_runtime_monitor.py
@@ -0,0 +1,88 @@
+import asyncio
+import unittest
+from unittest.mock import patch
+
+from src.lib.tracing import LocalTraceQueue, TraceEventType, TraceTracker
+from src.lib.tracing import runtime as tracing_runtime
+from src.lib.tracing.runtime import HealthProbe, RuntimeMonitor
+
+
+class RuntimeMonitorTests(unittest.TestCase):
+ def setUp(self):
+ self.original_tracer = tracing_runtime.tracer
+ self.tracker = TraceTracker(queue=LocalTraceQueue(), enabled=True)
+ tracing_runtime.tracer = self.tracker
+
+ def tearDown(self):
+ tracing_runtime.tracer = self.original_tracer
+ tracing_runtime._monitors.clear()
+
+ def test_runtime_monitor_records_heartbeat_resources_and_shutdown(self):
+ monitor = RuntimeMonitor(
+ service_name="test-service",
+ probes=[],
+ resource_sampler=lambda: {"rss_kb": 12},
+ )
+
+ monitor.heartbeat()
+ monitor.sample_resources()
+ monitor.stop()
+
+ events = self.tracker.queue.drain()
+ self.assertEqual(events[0].event_type, TraceEventType.HEARTBEAT)
+ self.assertEqual(events[1].event_type, TraceEventType.RESOURCE_SAMPLE)
+ self.assertEqual(events[1].metadata["rss_kb"], 12)
+ self.assertEqual(events[2].event_type, TraceEventType.PROCESS)
+ self.assertEqual(events[2].name, "process.stopped")
+
+ def test_health_check_records_success_and_downtime(self):
+ monitor = RuntimeMonitor(
+ service_name="test-service",
+ probes=[
+ HealthProbe("up", "127.0.0.1", 1),
+ HealthProbe("down", "127.0.0.1", 2),
+ ],
+ )
+
+ with patch.object(tracing_runtime, "probe_tcp", side_effect=[True, False]):
+ monitor.check_health()
+
+ events = self.tracker.queue.drain()
+ self.assertEqual(events[0].event_type, TraceEventType.HEALTH_CHECK)
+ self.assertEqual(events[0].name, "health.up")
+ self.assertEqual(events[1].event_type, TraceEventType.DOWNTIME)
+ self.assertEqual(events[1].name, "health.down.down")
+
+ def test_unhandled_exception_hook_records_exception(self):
+ monitor = RuntimeMonitor(service_name="test-service", probes=[])
+ exc = RuntimeError("boom")
+
+ monitor.record_unhandled_exception(exc, exc.__traceback__, "unit-test")
+
+ events = self.tracker.queue.drain()
+ self.assertEqual(len(events), 1)
+ self.assertEqual(events[0].event_type, TraceEventType.EXCEPTION)
+ self.assertEqual(events[0].severity.value, "critical")
+ self.assertEqual(events[0].operation, "unit-test")
+
+ def test_asyncio_exception_handler_records_context(self):
+ async def run_test():
+ loop = asyncio.get_running_loop()
+ previous_handler = loop.get_exception_handler()
+ loop.set_exception_handler(lambda _loop, _context: None)
+ monitor = RuntimeMonitor(service_name="test-service", probes=[])
+ monitor.install_asyncio_exception_handler(loop)
+ try:
+ loop.call_exception_handler({"message": "async problem"})
+ finally:
+ loop.set_exception_handler(previous_handler)
+
+ asyncio.run(run_test())
+
+ events = self.tracker.queue.drain()
+ self.assertEqual(events[0].event_type, TraceEventType.ERROR)
+ self.assertEqual(events[0].name, "runtime.asyncio.unhandled_error")
+
+
+if __name__ == "__main__":
+ unittest.main()