Skip to content

BrainAPI is a knowledge graph–powered AI memory layer that transforms unstructured data into structured knowledge, enabling intelligent search, recommendations, and contextual memory for AI agents and applications.

Trace agent loops and worker tasks

Cursor Agente62cfc886e3d
diff --git a/src/core/agents/core/invoke_loop.py b/src/core/agents/core/invoke_loop.py
index 406ed2c..a27c96e 100644
--- a/src/core/agents/core/invoke_loop.py
+++ b/src/core/agents/core/invoke_loop.py
@@ -1,3 +1,6 @@
+import os
+
+from src.lib.tracing import TraceSeverity, tracer
from src.utils.cleanup import strip_json
from .parsing import (
@@ -59,7 +62,51 @@ def _log_token_usage(response) -> None:
)
+def _trace_metadata(agent, config) -> dict:
+ metadata = (config or {}).get("metadata") or {}
+ return {
+ "agent": metadata.get("agent") or agent.__class__.__name__,
+ "brain_id": metadata.get("brain_id") or (config or {}).get("brain_id"),
+ "tags": (config or {}).get("tags") or [],
+ "tools_count": len(getattr(agent, "tools", []) or []),
+ "has_output_schema": agent.output_schema is not None,
+ }
+
+
def run_invoke_loop(agent, messages, config):
+ trace_metadata = _trace_metadata(agent, config)
+ trace_tenant_id = trace_metadata.get("brain_id")
+ with tracer.span(
+ "agent.invoke_loop",
+ service="brainapi-agent",
+ operation="invoke_loop",
+ tenant_id=trace_tenant_id,
+ metadata=trace_metadata,
+ slow_operation_ms=float(os.getenv("TRACE_AGENT_SLOW_OPERATION_MS", "30000")),
+ ):
+ return _run_invoke_loop_impl(
+ agent,
+ messages,
+ config,
+ trace_metadata=trace_metadata,
+ trace_tenant_id=trace_tenant_id,
+ )
+
+
+def _run_invoke_loop_impl(
+ agent,
+ messages,
+ config,
+ *,
+ trace_metadata: dict,
+ trace_tenant_id: str | None,
+):
+ agent_loop_threshold = int(os.getenv("TRACE_AGENT_LOOP_ITERATIONS", "20"))
+ tool_loop_threshold = int(os.getenv("TRACE_AGENT_TOOL_LOOP_ITERATIONS", "20"))
+ outer_loop_count = 0
+ tool_loop_count = 0
+ model_invoke_attempt_count = 0
+ schema_retry_count = 0
model_responses = []
agent.messages = []
agent.messages.append(
@@ -92,6 +139,16 @@ def run_invoke_loop(agent, messages, config):
structured_response = None
_schema_retry_count = 0
while True:
+ outer_loop_count += 1
+ tracer.expensive_loop(
+ "agent.invoke_loop.outer_loop",
+ outer_loop_count,
+ service="brainapi-agent",
+ operation="invoke_loop",
+ tenant_id=trace_tenant_id,
+ metadata=trace_metadata,
+ threshold=agent_loop_threshold,
+ )
_did_retry_recovered_tool_call = False
while n_message is None:
_did_retry_unknown_finish = False
@@ -120,9 +177,21 @@ def run_invoke_loop(agent, messages, config):
else:
_invoke_attempts = 3
for _invoke_attempt in range(_invoke_attempts):
+ model_invoke_attempt_count += 1
try:
_n_message = agent.model.invoke(agent.messages, config)
except Exception as _invoke_exc:
+ tracer.exception(
+ "agent.model.invoke.failed",
+ _invoke_exc,
+ service="brainapi-agent",
+ operation="model.invoke",
+ tenant_id=trace_tenant_id,
+ metadata={
+ **trace_metadata,
+ "attempt": _invoke_attempt + 1,
+ },
+ )
_exc_str = str(_invoke_exc).lower()
if agent._tools_bound and (
"invalid message format" in _exc_str
@@ -355,6 +424,19 @@ def run_invoke_loop(agent, messages, config):
_did_retry_recovered_next_tool_call = False
_last_called_tool_name = tool_name
while True:
+ tool_loop_count += 1
+ tracer.expensive_loop(
+ "agent.invoke_loop.tool_loop",
+ tool_loop_count,
+ service="brainapi-agent",
+ operation="tool_loop",
+ tenant_id=trace_tenant_id,
+ metadata={
+ **trace_metadata,
+ "last_tool_name": _last_called_tool_name,
+ },
+ threshold=tool_loop_threshold,
+ )
_next_attempts = 3
for _next_attempt in range(_next_attempts):
if (
@@ -386,6 +468,18 @@ def run_invoke_loop(agent, messages, config):
try:
next_response = agent.model.invoke(agent.messages, config)
except Exception as _next_exc:
+ tracer.exception(
+ "agent.model.next_invoke.failed",
+ _next_exc,
+ service="brainapi-agent",
+ operation="model.next_invoke",
+ tenant_id=trace_tenant_id,
+ metadata={
+ **trace_metadata,
+ "attempt": _next_attempt + 1,
+ "last_tool_name": _last_called_tool_name,
+ },
+ )
_next_exc_str = str(_next_exc).lower()
if agent._tools_bound and (
"invalid message format" in _next_exc_str
@@ -739,6 +833,20 @@ def run_invoke_loop(agent, messages, config):
)
break
except Exception as e:
+ schema_retry_count += 1
+ tracer.error(
+ "agent.structured_output.parse_failed",
+ service="brainapi-agent",
+ operation="structured_output.parse",
+ tenant_id=trace_tenant_id,
+ severity=TraceSeverity.WARNING,
+ message=str(e),
+ metadata={
+ **trace_metadata,
+ "schema_retry_count": schema_retry_count,
+ "error_type": type(e).__name__,
+ },
+ )
if agent.debug:
print("[DEBUG (agent_base)]: ", type(e).__name__, e)
print(
@@ -761,6 +869,16 @@ def run_invoke_loop(agent, messages, config):
n_message = None
continue
+ tracer.expensive_loop(
+ "agent.model.invoke_attempts",
+ model_invoke_attempt_count,
+ service="brainapi-agent",
+ operation="model.invoke",
+ tenant_id=trace_tenant_id,
+ metadata=trace_metadata,
+ threshold=int(os.getenv("TRACE_AGENT_MODEL_INVOKE_ATTEMPTS", "10")),
+ )
+
try:
from langchain_core.tracers.langchain import wait_for_all_tracers
diff --git a/src/lib/tracing/__init__.py b/src/lib/tracing/__init__.py
index 62f84bf..6ee1544 100644
--- a/src/lib/tracing/__init__.py
+++ b/src/lib/tracing/__init__.py
@@ -4,7 +4,7 @@ from src.lib.tracing.events import (
TraceSeverity,
)
from src.lib.tracing.subscribers import TraceSubscriber
-from src.lib.tracing.tracker import LocalTraceQueue, TraceTracker, trace_tracker
+from src.lib.tracing.tracker import LocalTraceQueue, TraceTracker, trace_tracker, tracer
__all__ = [
"LocalTraceQueue",
@@ -14,4 +14,5 @@ __all__ = [
"TraceSubscriber",
"TraceTracker",
"trace_tracker",
+ "tracer",
]
diff --git a/src/lib/tracing/tracker.py b/src/lib/tracing/tracker.py
index 134da40..77f6fc1 100644
--- a/src/lib/tracing/tracker.py
+++ b/src/lib/tracing/tracker.py
@@ -245,8 +245,14 @@ class TraceTracker:
tenant_id: Optional[str] = None,
trace_id: Optional[str] = None,
metadata: Optional[dict[str, Any]] = None,
+ threshold: Optional[int] = None,
) -> Optional[TraceEvent]:
- if iterations < self.expensive_loop_iterations:
+ limit = (
+ threshold
+ if threshold is not None
+ else self.expensive_loop_iterations
+ )
+ if iterations < limit:
return None
return self.publish(
TraceEventType.EXPENSIVE_LOOP,
@@ -268,6 +274,8 @@ class TraceTracker:
trace_id: Optional[str] = None,
metadata: Optional[dict[str, Any]] = None,
slow_operation_ms: Optional[float] = None,
+ service: Optional[str] = None,
+ operation: Optional[str] = None,
) -> Iterator[None]:
tokens = self.set_context(trace_id=trace_id, tenant_id=tenant_id)
started_at = time.perf_counter()
@@ -278,6 +286,8 @@ class TraceTracker:
self.exception(
name,
exc,
+ service=service,
+ operation=operation,
metadata={**(metadata or {}), "duration_ms": duration_ms},
)
raise
@@ -293,6 +303,8 @@ class TraceTracker:
TraceEventType.SLA_BREACH,
name,
severity=TraceSeverity.WARNING,
+ service=service,
+ operation=operation,
duration_ms=duration_ms,
threshold_ms=threshold,
metadata=metadata,
@@ -320,6 +332,7 @@ class TraceTracker:
tenant_id=tenant_id,
trace_id=trace_id,
metadata=metadata,
+ threshold=threshold,
)
async def dispatch_once(self) -> TraceEvent:
diff --git a/src/lib/tracing/workers.py b/src/lib/tracing/workers.py
new file mode 100644
index 0000000..95fe9ec
--- /dev/null
+++ b/src/lib/tracing/workers.py
@@ -0,0 +1,127 @@
+import time
+from typing import Any
+
+from celery import signals
+
+from src.lib.tracing.tracker import tracer
+
+_task_starts: dict[str, float] = {}
+_task_context_tokens: dict[str, Any] = {}
+
+
+def _task_id(task_id: str | None, task=None) -> str:
+ if task_id:
+ return task_id
+ request = getattr(task, "request", None)
+ return getattr(request, "id", None) or "unknown"
+
+
+def _task_name(task=None, sender=None) -> str:
+ return getattr(task, "name", None) or getattr(sender, "name", None) or "unknown"
+
+
+def _tenant_id(args=None, kwargs=None) -> str | None:
+ args = args or ()
+ kwargs = kwargs or {}
+ candidates = list(args)
+ candidates.extend(kwargs.values())
+ for candidate in candidates:
+ if isinstance(candidate, dict):
+ brain_id = candidate.get("brain_id")
+ if brain_id:
+ return str(brain_id)
+ brain_id = kwargs.get("brain_id")
+ return str(brain_id) if brain_id else None
+
+
+def _trace_id(task_id: str, kwargs=None) -> str:
+ kwargs = kwargs or {}
+ trace_id = kwargs.get("trace_id")
+ return str(trace_id) if trace_id else task_id
+
+
+def _metadata(task_id: str, task_name: str, args=None, kwargs=None) -> dict[str, Any]:
+ return {
+ "task_id": task_id,
+ "task_name": task_name,
+ "args_count": len(args or ()),
+ "kwargs_keys": sorted((kwargs or {}).keys()),
+ }
+
+
+def install_celery_tracing(_app=None, service_name: str = "brainapi-worker") -> None:
+ signals.task_prerun.connect(
+ _trace_task_prerun,
+ weak=False,
+ dispatch_uid=f"{service_name}.trace.task_prerun",
+ )
+ signals.task_postrun.connect(
+ _trace_task_postrun,
+ weak=False,
+ dispatch_uid=f"{service_name}.trace.task_postrun",
+ )
+ signals.task_failure.connect(
+ _trace_task_failure,
+ weak=False,
+ dispatch_uid=f"{service_name}.trace.task_failure",
+ )
+
+
+def _trace_task_prerun(sender=None, task_id=None, task=None, args=None, kwargs=None, **_):
+ resolved_task_id = _task_id(task_id, task)
+ resolved_task_name = _task_name(task, sender)
+ _task_starts[resolved_task_id] = time.perf_counter()
+ _task_context_tokens[resolved_task_id] = tracer.set_context(
+ trace_id=_trace_id(resolved_task_id, kwargs),
+ tenant_id=_tenant_id(args, kwargs),
+ )
+ tracer.publish(
+ "latency",
+ f"{resolved_task_name}.started",
+ service="brainapi-worker",
+ operation=resolved_task_name,
+ metadata=_metadata(resolved_task_id, resolved_task_name, args, kwargs),
+ )
+
+
+def _trace_task_postrun(sender=None, task_id=None, task=None, args=None, kwargs=None, retval=None, state=None, **_):
+ resolved_task_id = _task_id(task_id, task)
+ resolved_task_name = _task_name(task, sender)
+ started_at = _task_starts.pop(resolved_task_id, None)
+ duration_ms = (time.perf_counter() - started_at) * 1000 if started_at else None
+ metadata = {
+ **_metadata(resolved_task_id, resolved_task_name, args, kwargs),
+ "state": state,
+ "returned": retval is not None,
+ }
+ if duration_ms is not None and duration_ms >= tracer.default_sla_ms:
+ tracer.publish(
+ "sla_breach",
+ f"{resolved_task_name}.sla_breach",
+ service="brainapi-worker",
+ operation=resolved_task_name,
+ duration_ms=duration_ms,
+ threshold_ms=tracer.default_sla_ms,
+ metadata=metadata,
+ )
+ token = _task_context_tokens.pop(resolved_task_id, None)
+ if token is not None:
+ tracer.reset_context(token)
+
+
+def _trace_task_failure(sender=None, task_id=None, exception=None, args=None, kwargs=None, einfo=None, **_):
+ resolved_task_id = _task_id(task_id, sender)
+ resolved_task_name = _task_name(sender=sender)
+ started_at = _task_starts.get(resolved_task_id)
+ duration_ms = (time.perf_counter() - started_at) * 1000 if started_at else None
+ metadata = _metadata(resolved_task_id, resolved_task_name, args, kwargs)
+ if einfo is not None:
+ metadata["traceback"] = str(einfo)
+ tracer.exception(
+ f"{resolved_task_name}.failed",
+ exception or RuntimeError("Celery task failed"),
+ service="brainapi-worker",
+ operation=resolved_task_name,
+ duration_ms=duration_ms,
+ metadata=metadata,
+ )
diff --git a/src/workers/app.py b/src/workers/app.py
index fdaf412..aa4e75b 100644
--- a/src/workers/app.py
+++ b/src/workers/app.py
@@ -31,6 +31,7 @@ from celery import Celery
from kombu import Queue
from src.config import config
+from src.lib.tracing.workers import register_celery_tracing
os.environ.setdefault("GRPC_ENABLE_FORK_SUPPORT", "1")
@@ -98,3 +99,4 @@ ingestion_app.conf.update(
"fanout_patterns": True,
},
)
+register_celery_tracing(service_name="brainapi-worker")
diff --git a/tests/test_tracing_sla_tracker.py b/tests/test_tracing_sla_tracker.py
index ed47b2a..612ef5c 100644
--- a/tests/test_tracing_sla_tracker.py
+++ b/tests/test_tracing_sla_tracker.py
@@ -1,4 +1,5 @@
import asyncio
+from unittest.mock import patch
import unittest
from starlette.applications import Starlette
@@ -15,8 +16,10 @@ from src.lib.tracing import (
TraceSubscriber,
TraceTracker,
)
+from src.core.agents.core.invoke_loop import run_invoke_loop
from src.lib.tracing.middleware import TraceMiddleware
from src.lib.tracing import middleware as tracing_middleware
+from src.lib.tracing import workers as tracing_workers
class MemorySubscriber(TraceSubscriber):
diff --git a/tests/test_tracing_workers_and_agents.py b/tests/test_tracing_workers_and_agents.py
new file mode 100644
index 0000000..4913fa8
--- /dev/null
+++ b/tests/test_tracing_workers_and_agents.py
@@ -0,0 +1,140 @@
+import os
+import unittest
+from unittest.mock import patch
+
+from src.lib.tracing import LocalTraceQueue, TraceEventType, TraceTracker
+from src.lib.tracing import workers as tracing_workers
+
+
+class WorkerTracingTests(unittest.TestCase):
+ def setUp(self):
+ self.original_tracer = tracing_workers.tracer
+ self.tracker = TraceTracker(queue=LocalTraceQueue(), enabled=True)
+ tracing_workers.tracer = self.tracker
+
+ def tearDown(self):
+ tracing_workers.tracer = self.original_tracer
+ tracing_workers._task_starts.clear()
+ tracing_workers._task_context_tokens.clear()
+
+ def test_worker_signals_record_start_sla_and_failure(self):
+ tracing_workers._trace_task_prerun(
+ sender=type("Task", (), {"name": "demo.task"})(),
+ task_id="task-1",
+ args=({"brain_id": "tenant-1"},),
+ kwargs={},
+ )
+ tracing_workers._task_starts["task-1"] -= 2
+ tracing_workers._trace_task_failure(
+ sender=type("Task", (), {"name": "demo.task"})(),
+ task_id="task-1",
+ exception=RuntimeError("boom"),
+ args=({"brain_id": "tenant-1"},),
+ kwargs={},
+ )
+ tracing_workers._trace_task_postrun(
+ sender=type("Task", (), {"name": "demo.task"})(),
+ task_id="task-1",
+ args=({"brain_id": "tenant-1"},),
+ kwargs={},
+ state="FAILURE",
+ )
+
+ events = self.tracker.queue.drain()
+ self.assertEqual(events[0].event_type, TraceEventType.LATENCY)
+ self.assertEqual(events[0].tenant_id, "tenant-1")
+ self.assertEqual(events[1].event_type, TraceEventType.EXCEPTION)
+ self.assertEqual(events[1].error_type, "RuntimeError")
+ self.assertEqual(events[2].event_type, TraceEventType.SLA_BREACH)
+ self.assertEqual(events[2].operation, "demo.task")
+
+
+class AgentLoopTracingTests(unittest.TestCase):
+ def test_agent_loop_records_model_exception(self):
+ from src.core.agents.core import invoke_loop
+
+ tracker = TraceTracker(queue=LocalTraceQueue(), enabled=True)
+
+ class FailingModel:
+ def invoke(self, *_args, **_kwargs):
+ raise RuntimeError("model down")
+
+ class Agent:
+ class Tool:
+ name = "demo_tool"
+ description = "Demo tool"
+ args_schema = {}
+
+ tools = [Tool()]
+ output_schema = None
+ model = FailingModel()
+ thinking = False
+ _tools_bound = False
+ system_prompt = "system"
+ debug = False
+
+ def _model_requires_thought_signatures(self):
+ return False
+
+ with patch.object(invoke_loop, "tracer", tracker):
+ with self.assertRaises(RuntimeError):
+ invoke_loop.run_invoke_loop(
+ Agent(),
+ [{"role": "user", "content": "hello"}],
+ {"metadata": {"brain_id": "tenant-2", "agent": "test-agent"}},
+ )
+
+ events = tracker.queue.drain()
+ event_types = [event.event_type for event in events]
+ self.assertIn(TraceEventType.EXCEPTION, event_types)
+ self.assertTrue(
+ any(event.name == "agent.model.invoke.failed" for event in events)
+ )
+ self.assertTrue(any(event.name == "agent.invoke_loop" for event in events))
+ self.assertTrue(all(
+ event.tenant_id == "tenant-2" for event in events if event.tenant_id
+ ))
+
+ def test_agent_loop_records_outer_loop_threshold(self):
+ from src.core.agents.core import invoke_loop
+
+ tracker = TraceTracker(queue=LocalTraceQueue(), enabled=True)
+
+ class SimpleModel:
+ def invoke(self, *_args, **_kwargs):
+ return {"content": "done"}
+
+ class Agent:
+ tools = []
+ output_schema = None
+ model = SimpleModel()
+ thinking = False
+ _tools_bound = False
+ system_prompt = "system"
+ debug = False
+ _get_effective_output_schema = None
+
+ def _model_requires_thought_signatures(self):
+ return False
+
+ with patch.object(invoke_loop, "tracer", tracker):
+ with patch.dict(os.environ, {"TRACE_AGENT_LOOP_ITERATIONS": "1"}):
+ result = invoke_loop.run_invoke_loop(
+ Agent(),
+ [{"role": "user", "content": "hello"}],
+ {"metadata": {"brain_id": "tenant-3", "agent": "test-agent"}},
+ )
+
+ self.assertGreaterEqual(len(result["messages"]), 1)
+ events = tracker.queue.drain()
+ self.assertTrue(
+ any(
+ event.event_type == TraceEventType.EXPENSIVE_LOOP
+ and event.name == "agent.invoke_loop.outer_loop"
+ for event in events
+ )
+ )
+
+
+if __name__ == "__main__":
+ unittest.main()