Skip to content

Framework for orchestrating role-playing, autonomous AI agents. By fostering collaborative intelligence, CrewAI empowers agents to work together seamlessly, tackling complex tasks.

fix(telemetry): attribute tool errors to the tool that failed (#7043)

João Mourab1c9c89ad038
diff --git a/lib/crewai/src/crewai/tools/tool_usage.py b/lib/crewai/src/crewai/tools/tool_usage.py
index f87a170ac..573c709cb 100644
--- a/lib/crewai/src/crewai/tools/tool_usage.py
+++ b/lib/crewai/src/crewai/tools/tool_usage.py
@@ -331,7 +331,10 @@ class ToolUsage:
message=usage_limit_error,
reason=ToolFailureReason.USAGE_LIMIT,
)
- self._telemetry.tool_usage_error(llm=self.function_calling_llm)
+ self._telemetry.tool_usage_error(
+ llm=self.function_calling_llm,
+ tool_name=sanitize_tool_name(tool.name),
+ )
result = self._format_result(result=result)
elif result is None:
try:
@@ -450,7 +453,10 @@ class ToolUsage:
error_event_emitted = True
self._run_attempts += 1
if self._run_attempts > self._max_parsing_attempts:
- self._telemetry.tool_usage_error(llm=self.function_calling_llm)
+ self._telemetry.tool_usage_error(
+ llm=self.function_calling_llm,
+ tool_name=sanitize_tool_name(tool.name),
+ )
error_message = I18N_DEFAULT.errors(
"tool_usage_exception"
).format(
@@ -582,7 +588,10 @@ class ToolUsage:
message=usage_limit_error,
reason=ToolFailureReason.USAGE_LIMIT,
)
- self._telemetry.tool_usage_error(llm=self.function_calling_llm)
+ self._telemetry.tool_usage_error(
+ llm=self.function_calling_llm,
+ tool_name=sanitize_tool_name(tool.name),
+ )
result = self._format_result(result=result)
elif result is None:
try:
@@ -701,7 +710,10 @@ class ToolUsage:
error_event_emitted = True
self._run_attempts += 1
if self._run_attempts > self._max_parsing_attempts:
- self._telemetry.tool_usage_error(llm=self.function_calling_llm)
+ self._telemetry.tool_usage_error(
+ llm=self.function_calling_llm,
+ tool_name=sanitize_tool_name(tool.name),
+ )
error_message = I18N_DEFAULT.errors(
"tool_usage_exception"
).format(
@@ -910,6 +922,10 @@ class ToolUsage:
except Exception as e:
self._run_attempts += 1
if self._run_attempts > self._max_parsing_attempts:
+ # No tool_name here, deliberately: this is a tool-CALL PARSING failure, so the
+ # tool the model wanted was never identified. Passing tool_string would put raw,
+ # unbounded model output into a metrics dimension. Parse failures are counted as
+ # tool errors with an empty name, which is the honest representation.
self._telemetry.tool_usage_error(llm=self.function_calling_llm)
if self.task:
self.task.increment_tools_errors()
diff --git a/lib/crewai/tests/tools/test_tool_usage_error_attribution.py b/lib/crewai/tests/tools/test_tool_usage_error_attribution.py
new file mode 100644
index 000000000..3be9aee01
--- /dev/null
+++ b/lib/crewai/tests/tools/test_tool_usage_error_attribution.py
@@ -0,0 +1,145 @@
+"""`Tool Usage Error` spans must carry the tool name.
+
+The emitter has accepted `tool_name` since it was written
+(`Telemetry.tool_usage_error`, which writes the attribute only `if tool_name:`), but no caller
+passed it, so tool errors were attributed to the empty string and no per-tool error rate could
+be computed at all. The omission is silent by construction - a missing kwarg simply produces a
+span without the attribute - so it needs tests rather than review attention.
+
+Both error paths exist twice, once in the sync `_use` and once in the async `_ause`, so the
+matrix here is (usage-limit, execution-error) x (sync, async). The parsing failure in
+`_tool_calling` is covered too, pinning the opposite expectation: it must stay unattributed.
+"""
+
+from unittest.mock import MagicMock
+
+import pytest
+
+from crewai.tools import BaseTool
+from crewai.tools.tool_usage import ToolUsage
+from crewai.tools.tool_calling import ToolCalling
+from crewai.utilities.string_utils import sanitize_tool_name
+
+
+class ExplodingTool(BaseTool):
+ name: str = "Exploding Tool"
+ description: str = "Raises on every call, to exercise the execution-error path"
+
+ def _run(self, text: str) -> str:
+ raise RuntimeError("boom")
+
+
+class LimitedTool(BaseTool):
+ name: str = "Limited Tool"
+ description: str = "Has a usage limit, to exercise the usage-limit path"
+ max_usage_count: int = 1
+
+ def _run(self, text: str) -> str:
+ return f"ok {text}"
+
+
+def build_usage(tool: BaseTool) -> ToolUsage:
+ """A ToolUsage wired to one tool, with a mocked telemetry sink."""
+ # Real strings on the mocks: ToolUsageStartedEvent is a pydantic model and rejects
+ # MagicMock attributes, so a bare MagicMock() fails validation before the code under
+ # test is reached.
+ task = MagicMock()
+ task.name = "some task"
+ agent = MagicMock()
+ agent.role = "some role"
+ agent.key = "agent-key"
+ agent._original_role = "some role"
+ agent.verbose = False
+ action = MagicMock()
+ action.tool = tool.name
+ action.tool_input = {"text": "x"}
+
+ usage = ToolUsage(
+ tools_handler=MagicMock(cache=None, last_used_tool=None),
+ tools=[tool],
+ task=task,
+ function_calling_llm=None,
+ agent=agent,
+ action=action,
+ )
+ usage._telemetry = MagicMock()
+ # Report the error on the first failure instead of after the retry budget.
+ usage._max_parsing_attempts = 0
+ return usage
+
+
+def recorded_tool_name(usage: ToolUsage) -> str | None:
+ """The tool_name the code under test handed to telemetry."""
+ assert usage._telemetry.tool_usage_error.called, (
+ "expected a Tool Usage Error span to be emitted"
+ )
+ return usage._telemetry.tool_usage_error.call_args.kwargs.get("tool_name")
+
+
+def calling_for(tool: BaseTool) -> ToolCalling:
+ return ToolCalling(tool_name=tool.name, arguments={"text": "x"})
+
+
+def test_sync_execution_error_reports_the_tool_name():
+ tool = ExplodingTool()
+ usage = build_usage(tool)
+
+ usage.use(calling=calling_for(tool), tool_string="Action: Exploding Tool")
+
+ recorded = recorded_tool_name(usage)
+ assert recorded == sanitize_tool_name(tool.name)
+ assert recorded, "an empty tool_name reproduces the bug: the emitter skips falsy values"
+
+
+async def test_async_execution_error_reports_the_tool_name():
+ tool = ExplodingTool()
+ usage = build_usage(tool)
+
+ await usage.ause(calling=calling_for(tool), tool_string="Action: Exploding Tool")
+
+ recorded = recorded_tool_name(usage)
+ assert recorded == sanitize_tool_name(tool.name)
+ assert recorded, "an empty tool_name reproduces the bug: the emitter skips falsy values"
+
+
+def test_sync_usage_limit_error_reports_the_tool_name():
+ tool = LimitedTool()
+ tool.current_usage_count = tool.max_usage_count
+ usage = build_usage(tool)
+
+ usage.use(calling=calling_for(tool), tool_string="Action: Limited Tool")
+
+ recorded = recorded_tool_name(usage)
+ assert recorded == sanitize_tool_name(tool.name)
+ assert recorded, "an empty tool_name reproduces the bug: the emitter skips falsy values"
+
+
+async def test_async_usage_limit_error_reports_the_tool_name():
+ tool = LimitedTool()
+ tool.current_usage_count = tool.max_usage_count
+ usage = build_usage(tool)
+
+ await usage.ause(calling=calling_for(tool), tool_string="Action: Limited Tool")
+
+ recorded = recorded_tool_name(usage)
+ assert recorded == sanitize_tool_name(tool.name)
+ assert recorded, "an empty tool_name reproduces the bug: the emitter skips falsy values"
+
+
+def test_parsing_failure_stays_unattributed():
+ """A tool-call parsing failure has no tool identity, and must not invent one.
+
+ Passing the raw tool_string here would put unbounded model output into a metrics
+ dimension, which is worse than an honest empty name.
+ """
+ usage = build_usage(ExplodingTool())
+ # Force both parse attempts to raise. Without this the fallback returns a ToolUsageError
+ # object instead of raising, so the telemetry branch is never reached and the test would
+ # pass for the wrong reason.
+ usage._original_tool_calling = MagicMock(side_effect=RuntimeError("unparseable"))
+
+ usage._tool_calling("not a parseable tool call at all")
+
+ assert recorded_tool_name(usage) is None