Framework for orchestrating role-playing, autonomous AI agents. By fostering collaborative intelligence, CrewAI empowers agents to work together seamlessly, tackling complex tasks.
fix(telemetry): attribute tool errors to the tool that failed (#7043)
João Mourab1c9c89ad038
diff --git a/lib/crewai/src/crewai/tools/tool_usage.py b/lib/crewai/src/crewai/tools/tool_usage.pyindex f87a170ac..573c709cb 100644--- a/lib/crewai/src/crewai/tools/tool_usage.py+++ b/lib/crewai/src/crewai/tools/tool_usage.py@@ -331,7 +331,10 @@ class ToolUsage:message=usage_limit_error,reason=ToolFailureReason.USAGE_LIMIT,)- self._telemetry.tool_usage_error(llm=self.function_calling_llm)+ self._telemetry.tool_usage_error(+ llm=self.function_calling_llm,+ tool_name=sanitize_tool_name(tool.name),+ )result = self._format_result(result=result)elif result is None:try:@@ -450,7 +453,10 @@ class ToolUsage:error_event_emitted = Trueself._run_attempts += 1if self._run_attempts > self._max_parsing_attempts:- self._telemetry.tool_usage_error(llm=self.function_calling_llm)+ self._telemetry.tool_usage_error(+ llm=self.function_calling_llm,+ tool_name=sanitize_tool_name(tool.name),+ )error_message = I18N_DEFAULT.errors("tool_usage_exception").format(@@ -582,7 +588,10 @@ class ToolUsage:message=usage_limit_error,reason=ToolFailureReason.USAGE_LIMIT,)- self._telemetry.tool_usage_error(llm=self.function_calling_llm)+ self._telemetry.tool_usage_error(+ llm=self.function_calling_llm,+ tool_name=sanitize_tool_name(tool.name),+ )result = self._format_result(result=result)elif result is None:try:@@ -701,7 +710,10 @@ class ToolUsage:error_event_emitted = Trueself._run_attempts += 1if self._run_attempts > self._max_parsing_attempts:- self._telemetry.tool_usage_error(llm=self.function_calling_llm)+ self._telemetry.tool_usage_error(+ llm=self.function_calling_llm,+ tool_name=sanitize_tool_name(tool.name),+ )error_message = I18N_DEFAULT.errors("tool_usage_exception").format(@@ -910,6 +922,10 @@ class ToolUsage:except Exception as e:self._run_attempts += 1if self._run_attempts > self._max_parsing_attempts:+ # No tool_name here, deliberately: this is a tool-CALL PARSING failure, so the+ # tool the model wanted was never identified. Passing tool_string would put raw,+ # unbounded model output into a metrics dimension. Parse failures are counted as+ # tool errors with an empty name, which is the honest representation.self._telemetry.tool_usage_error(llm=self.function_calling_llm)if self.task:self.task.increment_tools_errors()diff --git a/lib/crewai/tests/tools/test_tool_usage_error_attribution.py b/lib/crewai/tests/tools/test_tool_usage_error_attribution.pynew file mode 100644index 000000000..3be9aee01--- /dev/null+++ b/lib/crewai/tests/tools/test_tool_usage_error_attribution.py@@ -0,0 +1,145 @@+"""`Tool Usage Error` spans must carry the tool name.++The emitter has accepted `tool_name` since it was written+(`Telemetry.tool_usage_error`, which writes the attribute only `if tool_name:`), but no caller+passed it, so tool errors were attributed to the empty string and no per-tool error rate could+be computed at all. The omission is silent by construction - a missing kwarg simply produces a+span without the attribute - so it needs tests rather than review attention.++Both error paths exist twice, once in the sync `_use` and once in the async `_ause`, so the+matrix here is (usage-limit, execution-error) x (sync, async). The parsing failure in+`_tool_calling` is covered too, pinning the opposite expectation: it must stay unattributed.+"""++from unittest.mock import MagicMock++import pytest++from crewai.tools import BaseTool+from crewai.tools.tool_usage import ToolUsage+from crewai.tools.tool_calling import ToolCalling+from crewai.utilities.string_utils import sanitize_tool_name+++class ExplodingTool(BaseTool):+ name: str = "Exploding Tool"+ description: str = "Raises on every call, to exercise the execution-error path"++ def _run(self, text: str) -> str:+ raise RuntimeError("boom")+++class LimitedTool(BaseTool):+ name: str = "Limited Tool"+ description: str = "Has a usage limit, to exercise the usage-limit path"+ max_usage_count: int = 1++ def _run(self, text: str) -> str:+ return f"ok {text}"+++def build_usage(tool: BaseTool) -> ToolUsage:+ """A ToolUsage wired to one tool, with a mocked telemetry sink."""+ # Real strings on the mocks: ToolUsageStartedEvent is a pydantic model and rejects+ # MagicMock attributes, so a bare MagicMock() fails validation before the code under+ # test is reached.+ task = MagicMock()+ task.name = "some task"+ agent = MagicMock()+ agent.role = "some role"+ agent.key = "agent-key"+ agent._original_role = "some role"+ agent.verbose = False+ action = MagicMock()+ action.tool = tool.name+ action.tool_input = {"text": "x"}++ usage = ToolUsage(+ tools_handler=MagicMock(cache=None, last_used_tool=None),+ tools=[tool],+ task=task,+ function_calling_llm=None,+ agent=agent,+ action=action,+ )+ usage._telemetry = MagicMock()+ # Report the error on the first failure instead of after the retry budget.+ usage._max_parsing_attempts = 0+ return usage+++def recorded_tool_name(usage: ToolUsage) -> str | None:+ """The tool_name the code under test handed to telemetry."""+ assert usage._telemetry.tool_usage_error.called, (+ "expected a Tool Usage Error span to be emitted"+ )+ return usage._telemetry.tool_usage_error.call_args.kwargs.get("tool_name")+++def calling_for(tool: BaseTool) -> ToolCalling:+ return ToolCalling(tool_name=tool.name, arguments={"text": "x"})+++def test_sync_execution_error_reports_the_tool_name():+ tool = ExplodingTool()+ usage = build_usage(tool)++ usage.use(calling=calling_for(tool), tool_string="Action: Exploding Tool")++ recorded = recorded_tool_name(usage)+ assert recorded == sanitize_tool_name(tool.name)+ assert recorded, "an empty tool_name reproduces the bug: the emitter skips falsy values"+++async def test_async_execution_error_reports_the_tool_name():+ tool = ExplodingTool()+ usage = build_usage(tool)++ await usage.ause(calling=calling_for(tool), tool_string="Action: Exploding Tool")++ recorded = recorded_tool_name(usage)+ assert recorded == sanitize_tool_name(tool.name)+ assert recorded, "an empty tool_name reproduces the bug: the emitter skips falsy values"+++def test_sync_usage_limit_error_reports_the_tool_name():+ tool = LimitedTool()+ tool.current_usage_count = tool.max_usage_count+ usage = build_usage(tool)++ usage.use(calling=calling_for(tool), tool_string="Action: Limited Tool")++ recorded = recorded_tool_name(usage)+ assert recorded == sanitize_tool_name(tool.name)+ assert recorded, "an empty tool_name reproduces the bug: the emitter skips falsy values"+++async def test_async_usage_limit_error_reports_the_tool_name():+ tool = LimitedTool()+ tool.current_usage_count = tool.max_usage_count+ usage = build_usage(tool)++ await usage.ause(calling=calling_for(tool), tool_string="Action: Limited Tool")++ recorded = recorded_tool_name(usage)+ assert recorded == sanitize_tool_name(tool.name)+ assert recorded, "an empty tool_name reproduces the bug: the emitter skips falsy values"+++def test_parsing_failure_stays_unattributed():+ """A tool-call parsing failure has no tool identity, and must not invent one.++ Passing the raw tool_string here would put unbounded model output into a metrics+ dimension, which is worse than an honest empty name.+ """+ usage = build_usage(ExplodingTool())+ # Force both parse attempts to raise. Without this the fallback returns a ToolUsageError+ # object instead of raising, so the telemetry branch is never reached and the test would+ # pass for the wrong reason.+ usage._original_tool_calling = MagicMock(side_effect=RuntimeError("unparseable"))++ usage._tool_calling("not a parseable tool call at all")++ assert recorded_tool_name(usage) is None