BrainAPI is a knowledge graph–powered AI memory layer that transforms unstructured data into structured knowledge, enabling intelligent search, recommendations, and contextual memory for AI agents and applications.
Python ai-agentsai-memoryknowledge-graphknowledge-managementknowledge-representationreccomendation-system
Trace agent loops and worker tasks
Cursor Agente62cfc886e3d
diff --git a/src/core/agents/core/invoke_loop.py b/src/core/agents/core/invoke_loop.pyindex 406ed2c..a27c96e 100644--- a/src/core/agents/core/invoke_loop.py+++ b/src/core/agents/core/invoke_loop.py@@ -1,3 +1,6 @@+import os++from src.lib.tracing import TraceSeverity, tracerfrom src.utils.cleanup import strip_jsonfrom .parsing import (@@ -59,7 +62,51 @@ def _log_token_usage(response) -> None:)+def _trace_metadata(agent, config) -> dict:+ metadata = (config or {}).get("metadata") or {}+ return {+ "agent": metadata.get("agent") or agent.__class__.__name__,+ "brain_id": metadata.get("brain_id") or (config or {}).get("brain_id"),+ "tags": (config or {}).get("tags") or [],+ "tools_count": len(getattr(agent, "tools", []) or []),+ "has_output_schema": agent.output_schema is not None,+ }++def run_invoke_loop(agent, messages, config):+ trace_metadata = _trace_metadata(agent, config)+ trace_tenant_id = trace_metadata.get("brain_id")+ with tracer.span(+ "agent.invoke_loop",+ service="brainapi-agent",+ operation="invoke_loop",+ tenant_id=trace_tenant_id,+ metadata=trace_metadata,+ slow_operation_ms=float(os.getenv("TRACE_AGENT_SLOW_OPERATION_MS", "30000")),+ ):+ return _run_invoke_loop_impl(+ agent,+ messages,+ config,+ trace_metadata=trace_metadata,+ trace_tenant_id=trace_tenant_id,+ )+++def _run_invoke_loop_impl(+ agent,+ messages,+ config,+ *,+ trace_metadata: dict,+ trace_tenant_id: str | None,+):+ agent_loop_threshold = int(os.getenv("TRACE_AGENT_LOOP_ITERATIONS", "20"))+ tool_loop_threshold = int(os.getenv("TRACE_AGENT_TOOL_LOOP_ITERATIONS", "20"))+ outer_loop_count = 0+ tool_loop_count = 0+ model_invoke_attempt_count = 0+ schema_retry_count = 0model_responses = []agent.messages = []agent.messages.append(@@ -92,6 +139,16 @@ def run_invoke_loop(agent, messages, config):structured_response = None_schema_retry_count = 0while True:+ outer_loop_count += 1+ tracer.expensive_loop(+ "agent.invoke_loop.outer_loop",+ outer_loop_count,+ service="brainapi-agent",+ operation="invoke_loop",+ tenant_id=trace_tenant_id,+ metadata=trace_metadata,+ threshold=agent_loop_threshold,+ )_did_retry_recovered_tool_call = Falsewhile n_message is None:_did_retry_unknown_finish = False@@ -120,9 +177,21 @@ def run_invoke_loop(agent, messages, config):else:_invoke_attempts = 3for _invoke_attempt in range(_invoke_attempts):+ model_invoke_attempt_count += 1try:_n_message = agent.model.invoke(agent.messages, config)except Exception as _invoke_exc:+ tracer.exception(+ "agent.model.invoke.failed",+ _invoke_exc,+ service="brainapi-agent",+ operation="model.invoke",+ tenant_id=trace_tenant_id,+ metadata={+ **trace_metadata,+ "attempt": _invoke_attempt + 1,+ },+ )_exc_str = str(_invoke_exc).lower()if agent._tools_bound and ("invalid message format" in _exc_str@@ -355,6 +424,19 @@ def run_invoke_loop(agent, messages, config):_did_retry_recovered_next_tool_call = False_last_called_tool_name = tool_namewhile True:+ tool_loop_count += 1+ tracer.expensive_loop(+ "agent.invoke_loop.tool_loop",+ tool_loop_count,+ service="brainapi-agent",+ operation="tool_loop",+ tenant_id=trace_tenant_id,+ metadata={+ **trace_metadata,+ "last_tool_name": _last_called_tool_name,+ },+ threshold=tool_loop_threshold,+ )_next_attempts = 3for _next_attempt in range(_next_attempts):if (@@ -386,6 +468,18 @@ def run_invoke_loop(agent, messages, config):try:next_response = agent.model.invoke(agent.messages, config)except Exception as _next_exc:+ tracer.exception(+ "agent.model.next_invoke.failed",+ _next_exc,+ service="brainapi-agent",+ operation="model.next_invoke",+ tenant_id=trace_tenant_id,+ metadata={+ **trace_metadata,+ "attempt": _next_attempt + 1,+ "last_tool_name": _last_called_tool_name,+ },+ )_next_exc_str = str(_next_exc).lower()if agent._tools_bound and ("invalid message format" in _next_exc_str@@ -739,6 +833,20 @@ def run_invoke_loop(agent, messages, config):)breakexcept Exception as e:+ schema_retry_count += 1+ tracer.error(+ "agent.structured_output.parse_failed",+ service="brainapi-agent",+ operation="structured_output.parse",+ tenant_id=trace_tenant_id,+ severity=TraceSeverity.WARNING,+ message=str(e),+ metadata={+ **trace_metadata,+ "schema_retry_count": schema_retry_count,+ "error_type": type(e).__name__,+ },+ )if agent.debug:print("[DEBUG (agent_base)]: ", type(e).__name__, e)print(@@ -761,6 +869,16 @@ def run_invoke_loop(agent, messages, config):n_message = Nonecontinue+ tracer.expensive_loop(+ "agent.model.invoke_attempts",+ model_invoke_attempt_count,+ service="brainapi-agent",+ operation="model.invoke",+ tenant_id=trace_tenant_id,+ metadata=trace_metadata,+ threshold=int(os.getenv("TRACE_AGENT_MODEL_INVOKE_ATTEMPTS", "10")),+ )+try:from langchain_core.tracers.langchain import wait_for_all_tracersdiff --git a/src/lib/tracing/__init__.py b/src/lib/tracing/__init__.pyindex 62f84bf..6ee1544 100644--- a/src/lib/tracing/__init__.py+++ b/src/lib/tracing/__init__.py@@ -4,7 +4,7 @@ from src.lib.tracing.events import (TraceSeverity,)from src.lib.tracing.subscribers import TraceSubscriber-from src.lib.tracing.tracker import LocalTraceQueue, TraceTracker, trace_tracker+from src.lib.tracing.tracker import LocalTraceQueue, TraceTracker, trace_tracker, tracer__all__ = ["LocalTraceQueue",@@ -14,4 +14,5 @@ __all__ = ["TraceSubscriber","TraceTracker","trace_tracker",+ "tracer",]diff --git a/src/lib/tracing/tracker.py b/src/lib/tracing/tracker.pyindex 134da40..77f6fc1 100644--- a/src/lib/tracing/tracker.py+++ b/src/lib/tracing/tracker.py@@ -245,8 +245,14 @@ class TraceTracker:tenant_id: Optional[str] = None,trace_id: Optional[str] = None,metadata: Optional[dict[str, Any]] = None,+ threshold: Optional[int] = None,) -> Optional[TraceEvent]:- if iterations < self.expensive_loop_iterations:+ limit = (+ threshold+ if threshold is not None+ else self.expensive_loop_iterations+ )+ if iterations < limit:return Nonereturn self.publish(TraceEventType.EXPENSIVE_LOOP,@@ -268,6 +274,8 @@ class TraceTracker:trace_id: Optional[str] = None,metadata: Optional[dict[str, Any]] = None,slow_operation_ms: Optional[float] = None,+ service: Optional[str] = None,+ operation: Optional[str] = None,) -> Iterator[None]:tokens = self.set_context(trace_id=trace_id, tenant_id=tenant_id)started_at = time.perf_counter()@@ -278,6 +286,8 @@ class TraceTracker:self.exception(name,exc,+ service=service,+ operation=operation,metadata={**(metadata or {}), "duration_ms": duration_ms},)raise@@ -293,6 +303,8 @@ class TraceTracker:TraceEventType.SLA_BREACH,name,severity=TraceSeverity.WARNING,+ service=service,+ operation=operation,duration_ms=duration_ms,threshold_ms=threshold,metadata=metadata,@@ -320,6 +332,7 @@ class TraceTracker:tenant_id=tenant_id,trace_id=trace_id,metadata=metadata,+ threshold=threshold,)async def dispatch_once(self) -> TraceEvent:diff --git a/src/lib/tracing/workers.py b/src/lib/tracing/workers.pynew file mode 100644index 0000000..95fe9ec--- /dev/null+++ b/src/lib/tracing/workers.py@@ -0,0 +1,127 @@+import time+from typing import Any++from celery import signals++from src.lib.tracing.tracker import tracer++_task_starts: dict[str, float] = {}+_task_context_tokens: dict[str, Any] = {}+++def _task_id(task_id: str | None, task=None) -> str:+ if task_id:+ return task_id+ request = getattr(task, "request", None)+ return getattr(request, "id", None) or "unknown"+++def _task_name(task=None, sender=None) -> str:+ return getattr(task, "name", None) or getattr(sender, "name", None) or "unknown"+++def _tenant_id(args=None, kwargs=None) -> str | None:+ args = args or ()+ kwargs = kwargs or {}+ candidates = list(args)+ candidates.extend(kwargs.values())+ for candidate in candidates:+ if isinstance(candidate, dict):+ brain_id = candidate.get("brain_id")+ if brain_id:+ return str(brain_id)+ brain_id = kwargs.get("brain_id")+ return str(brain_id) if brain_id else None+++def _trace_id(task_id: str, kwargs=None) -> str:+ kwargs = kwargs or {}+ trace_id = kwargs.get("trace_id")+ return str(trace_id) if trace_id else task_id+++def _metadata(task_id: str, task_name: str, args=None, kwargs=None) -> dict[str, Any]:+ return {+ "task_id": task_id,+ "task_name": task_name,+ "args_count": len(args or ()),+ "kwargs_keys": sorted((kwargs or {}).keys()),+ }+++def install_celery_tracing(_app=None, service_name: str = "brainapi-worker") -> None:+ signals.task_prerun.connect(+ _trace_task_prerun,+ weak=False,+ dispatch_uid=f"{service_name}.trace.task_prerun",+ )+ signals.task_postrun.connect(+ _trace_task_postrun,+ weak=False,+ dispatch_uid=f"{service_name}.trace.task_postrun",+ )+ signals.task_failure.connect(+ _trace_task_failure,+ weak=False,+ dispatch_uid=f"{service_name}.trace.task_failure",+ )+++def _trace_task_prerun(sender=None, task_id=None, task=None, args=None, kwargs=None, **_):+ resolved_task_id = _task_id(task_id, task)+ resolved_task_name = _task_name(task, sender)+ _task_starts[resolved_task_id] = time.perf_counter()+ _task_context_tokens[resolved_task_id] = tracer.set_context(+ trace_id=_trace_id(resolved_task_id, kwargs),+ tenant_id=_tenant_id(args, kwargs),+ )+ tracer.publish(+ "latency",+ f"{resolved_task_name}.started",+ service="brainapi-worker",+ operation=resolved_task_name,+ metadata=_metadata(resolved_task_id, resolved_task_name, args, kwargs),+ )+++def _trace_task_postrun(sender=None, task_id=None, task=None, args=None, kwargs=None, retval=None, state=None, **_):+ resolved_task_id = _task_id(task_id, task)+ resolved_task_name = _task_name(task, sender)+ started_at = _task_starts.pop(resolved_task_id, None)+ duration_ms = (time.perf_counter() - started_at) * 1000 if started_at else None+ metadata = {+ **_metadata(resolved_task_id, resolved_task_name, args, kwargs),+ "state": state,+ "returned": retval is not None,+ }+ if duration_ms is not None and duration_ms >= tracer.default_sla_ms:+ tracer.publish(+ "sla_breach",+ f"{resolved_task_name}.sla_breach",+ service="brainapi-worker",+ operation=resolved_task_name,+ duration_ms=duration_ms,+ threshold_ms=tracer.default_sla_ms,+ metadata=metadata,+ )+ token = _task_context_tokens.pop(resolved_task_id, None)+ if token is not None:+ tracer.reset_context(token)+++def _trace_task_failure(sender=None, task_id=None, exception=None, args=None, kwargs=None, einfo=None, **_):+ resolved_task_id = _task_id(task_id, sender)+ resolved_task_name = _task_name(sender=sender)+ started_at = _task_starts.get(resolved_task_id)+ duration_ms = (time.perf_counter() - started_at) * 1000 if started_at else None+ metadata = _metadata(resolved_task_id, resolved_task_name, args, kwargs)+ if einfo is not None:+ metadata["traceback"] = str(einfo)+ tracer.exception(+ f"{resolved_task_name}.failed",+ exception or RuntimeError("Celery task failed"),+ service="brainapi-worker",+ operation=resolved_task_name,+ duration_ms=duration_ms,+ metadata=metadata,+ )diff --git a/src/workers/app.py b/src/workers/app.pyindex fdaf412..aa4e75b 100644--- a/src/workers/app.py+++ b/src/workers/app.py@@ -31,6 +31,7 @@ from celery import Celeryfrom kombu import Queuefrom src.config import config+from src.lib.tracing.workers import register_celery_tracingos.environ.setdefault("GRPC_ENABLE_FORK_SUPPORT", "1")@@ -98,3 +99,4 @@ ingestion_app.conf.update("fanout_patterns": True,},)+register_celery_tracing(service_name="brainapi-worker")diff --git a/tests/test_tracing_sla_tracker.py b/tests/test_tracing_sla_tracker.pyindex ed47b2a..612ef5c 100644--- a/tests/test_tracing_sla_tracker.py+++ b/tests/test_tracing_sla_tracker.py@@ -1,4 +1,5 @@import asyncio+from unittest.mock import patchimport unittestfrom starlette.applications import Starlette@@ -15,8 +16,10 @@ from src.lib.tracing import (TraceSubscriber,TraceTracker,)+from src.core.agents.core.invoke_loop import run_invoke_loopfrom src.lib.tracing.middleware import TraceMiddlewarefrom src.lib.tracing import middleware as tracing_middleware+from src.lib.tracing import workers as tracing_workersclass MemorySubscriber(TraceSubscriber):diff --git a/tests/test_tracing_workers_and_agents.py b/tests/test_tracing_workers_and_agents.pynew file mode 100644index 0000000..4913fa8--- /dev/null+++ b/tests/test_tracing_workers_and_agents.py@@ -0,0 +1,140 @@+import os+import unittest+from unittest.mock import patch++from src.lib.tracing import LocalTraceQueue, TraceEventType, TraceTracker+from src.lib.tracing import workers as tracing_workers+++class WorkerTracingTests(unittest.TestCase):+ def setUp(self):+ self.original_tracer = tracing_workers.tracer+ self.tracker = TraceTracker(queue=LocalTraceQueue(), enabled=True)+ tracing_workers.tracer = self.tracker++ def tearDown(self):+ tracing_workers.tracer = self.original_tracer+ tracing_workers._task_starts.clear()+ tracing_workers._task_context_tokens.clear()++ def test_worker_signals_record_start_sla_and_failure(self):+ tracing_workers._trace_task_prerun(+ sender=type("Task", (), {"name": "demo.task"})(),+ task_id="task-1",+ args=({"brain_id": "tenant-1"},),+ kwargs={},+ )+ tracing_workers._task_starts["task-1"] -= 2+ tracing_workers._trace_task_failure(+ sender=type("Task", (), {"name": "demo.task"})(),+ task_id="task-1",+ exception=RuntimeError("boom"),+ args=({"brain_id": "tenant-1"},),+ kwargs={},+ )+ tracing_workers._trace_task_postrun(+ sender=type("Task", (), {"name": "demo.task"})(),+ task_id="task-1",+ args=({"brain_id": "tenant-1"},),+ kwargs={},+ state="FAILURE",+ )++ events = self.tracker.queue.drain()+ self.assertEqual(events[0].event_type, TraceEventType.LATENCY)+ self.assertEqual(events[0].tenant_id, "tenant-1")+ self.assertEqual(events[1].event_type, TraceEventType.EXCEPTION)+ self.assertEqual(events[1].error_type, "RuntimeError")+ self.assertEqual(events[2].event_type, TraceEventType.SLA_BREACH)+ self.assertEqual(events[2].operation, "demo.task")+++class AgentLoopTracingTests(unittest.TestCase):+ def test_agent_loop_records_model_exception(self):+ from src.core.agents.core import invoke_loop++ tracker = TraceTracker(queue=LocalTraceQueue(), enabled=True)++ class FailingModel:+ def invoke(self, *_args, **_kwargs):+ raise RuntimeError("model down")++ class Agent:+ class Tool:+ name = "demo_tool"+ description = "Demo tool"+ args_schema = {}++ tools = [Tool()]+ output_schema = None+ model = FailingModel()+ thinking = False+ _tools_bound = False+ system_prompt = "system"+ debug = False++ def _model_requires_thought_signatures(self):+ return False++ with patch.object(invoke_loop, "tracer", tracker):+ with self.assertRaises(RuntimeError):+ invoke_loop.run_invoke_loop(+ Agent(),+ [{"role": "user", "content": "hello"}],+ {"metadata": {"brain_id": "tenant-2", "agent": "test-agent"}},+ )++ events = tracker.queue.drain()+ event_types = [event.event_type for event in events]+ self.assertIn(TraceEventType.EXCEPTION, event_types)+ self.assertTrue(+ any(event.name == "agent.model.invoke.failed" for event in events)+ )+ self.assertTrue(any(event.name == "agent.invoke_loop" for event in events))+ self.assertTrue(all(+ event.tenant_id == "tenant-2" for event in events if event.tenant_id+ ))++ def test_agent_loop_records_outer_loop_threshold(self):+ from src.core.agents.core import invoke_loop++ tracker = TraceTracker(queue=LocalTraceQueue(), enabled=True)++ class SimpleModel:+ def invoke(self, *_args, **_kwargs):+ return {"content": "done"}++ class Agent:+ tools = []+ output_schema = None+ model = SimpleModel()+ thinking = False+ _tools_bound = False+ system_prompt = "system"+ debug = False+ _get_effective_output_schema = None++ def _model_requires_thought_signatures(self):+ return False++ with patch.object(invoke_loop, "tracer", tracker):+ with patch.dict(os.environ, {"TRACE_AGENT_LOOP_ITERATIONS": "1"}):+ result = invoke_loop.run_invoke_loop(+ Agent(),+ [{"role": "user", "content": "hello"}],+ {"metadata": {"brain_id": "tenant-3", "agent": "test-agent"}},+ )++ self.assertGreaterEqual(len(result["messages"]), 1)+ events = tracker.queue.drain()+ self.assertTrue(+ any(+ event.event_type == TraceEventType.EXPENSIVE_LOOP+ and event.name == "agent.invoke_loop.outer_loop"+ for event in events+ )+ )+++if __name__ == "__main__":+ unittest.main()