Skip to content

The agent engineering platform.

feat(core): automatically count and store meta for tool call count (#33756)

Mason Daughertyed9bd6e3adb8
diff --git a/libs/core/langchain_core/tracers/core.py b/libs/core/langchain_core/tracers/core.py
index 97a2092d15..76a4a0356f 100644
--- a/libs/core/langchain_core/tracers/core.py
+++ b/libs/core/langchain_core/tracers/core.py
@@ -284,6 +284,16 @@ class _TracerCore(ABC):
llm_run.end_time = datetime.now(timezone.utc)
llm_run.events.append({"name": "end", "time": llm_run.end_time})
+ tool_call_count = 0
+ for generations in response.generations:
+ for generation in generations:
+ if hasattr(generation, "message"):
+ msg = generation.message
+ if hasattr(msg, "tool_calls"):
+ tool_call_count += len(msg.tool_calls)
+ if tool_call_count > 0:
+ llm_run.extra["tool_call_count"] = tool_call_count
+
return llm_run
def _errored_llm_run(
diff --git a/libs/core/tests/unit_tests/tracers/test_automatic_metadata.py b/libs/core/tests/unit_tests/tracers/test_automatic_metadata.py
new file mode 100644
index 0000000000..147997c2c1
--- /dev/null
+++ b/libs/core/tests/unit_tests/tracers/test_automatic_metadata.py
@@ -0,0 +1,128 @@
+"""Test automatic tool call count storage in tracers."""
+
+from __future__ import annotations
+
+from unittest.mock import MagicMock
+
+from langchain_core.messages import AIMessage
+from langchain_core.messages.tool import ToolCall
+from langchain_core.outputs import ChatGeneration, LLMResult
+from langchain_core.tracers.core import _TracerCore
+from langchain_core.tracers.schemas import Run
+
+
+class MockTracerCore(_TracerCore):
+ """Mock tracer core for testing LLM run completion."""
+
+ def __init__(self) -> None:
+ super().__init__()
+
+ def _persist_run(self, run: Run) -> None:
+ """Mock implementation of _persist_run."""
+
+
+def test_complete_llm_run_automatically_stores_tool_call_count() -> None:
+ """Test that `_complete_llm_run` automatically stores tool call count."""
+ tracer = MockTracerCore()
+
+ run = MagicMock(spec=Run)
+ run.id = "test-llm-run-id"
+ run.run_type = "llm"
+ run.extra = {}
+ run.outputs = {}
+ run.events = []
+ run.end_time = None
+ run.inputs = {}
+
+ tracer.run_map[str(run.id)] = run
+
+ tool_calls = [
+ ToolCall(name="search", args={"query": "test"}, id="call_1"),
+ ToolCall(name="calculator", args={"expression": "2+2"}, id="call_2"),
+ ]
+ message = AIMessage(content="Test", tool_calls=tool_calls)
+ generation = ChatGeneration(message=message)
+ response = LLMResult(generations=[[generation]])
+
+ # Complete the LLM run (this should trigger automatic metadata storage)
+ completed_run = tracer._complete_llm_run(response=response, run_id=run.id)
+
+ assert "tool_call_count" in completed_run.extra
+ assert completed_run.extra["tool_call_count"] == 2
+
+
+def test_complete_llm_run_handles_no_tool_calls() -> None:
+ """Test that `_complete_llm_run` handles runs with no tool calls gracefully."""
+ tracer = MockTracerCore()
+
+ run = MagicMock(spec=Run)
+ run.id = "test-llm-run-id-no-tools"
+ run.run_type = "llm"
+ run.extra = {}
+ run.outputs = {}
+ run.events = []
+ run.end_time = None
+ run.inputs = {}
+
+ tracer.run_map[str(run.id)] = run
+
+ message = AIMessage(content="No tools here")
+ generation = ChatGeneration(message=message)
+ response = LLMResult(generations=[[generation]])
+
+ completed_run = tracer._complete_llm_run(response=response, run_id=run.id)
+
+ # Verify tool call count is not stored when there are no tool calls
+ assert "tool_call_count" not in completed_run.extra
+
+
+def test_complete_llm_run_handles_empty_generations() -> None:
+ """Test that `_complete_llm_run` handles empty generations gracefully."""
+ tracer = MockTracerCore()
+
+ run = MagicMock(spec=Run)
+ run.id = "test-llm-run-id-empty"
+ run.run_type = "llm"
+ run.extra = {}
+ run.outputs = {}
+ run.events = []
+ run.end_time = None
+ run.inputs = {}
+
+ tracer.run_map[str(run.id)] = run
+
+ response = LLMResult(generations=[[]])
+
+ completed_run = tracer._complete_llm_run(response=response, run_id=run.id)
+
+ assert "tool_call_count" not in completed_run.extra
+
+
+def test_complete_llm_run_counts_tool_calls_from_multiple_generations() -> None:
+ """Test that tool calls are counted from multiple generations."""
+ tracer = MockTracerCore()
+
+ run = MagicMock(spec=Run)
+ run.id = "test-llm-run-id-multi"
+ run.run_type = "llm"
+ run.extra = {}
+ run.outputs = {}
+ run.events = []
+ run.end_time = None
+ run.inputs = {}
+
+ tracer.run_map[str(run.id)] = run
+
+ # Create multiple generations with tool calls
+ tool_calls_1 = [ToolCall(name="search", args={"query": "test"}, id="call_1")]
+ tool_calls_2 = [
+ ToolCall(name="calculator", args={"expression": "2+2"}, id="call_2"),
+ ToolCall(name="weather", args={"location": "NYC"}, id="call_3"),
+ ]
+ gen1 = ChatGeneration(message=AIMessage(content="Gen1", tool_calls=tool_calls_1))
+ gen2 = ChatGeneration(message=AIMessage(content="Gen2", tool_calls=tool_calls_2))
+ response = LLMResult(generations=[[gen1], [gen2]])
+
+ completed_run = tracer._complete_llm_run(response=response, run_id=run.id)
+
+ assert completed_run.extra["tool_call_count"] == 3
diff --git a/libs/core/tests/unit_tests/tracers/test_imports.py b/libs/core/tests/unit_tests/tracers/test_imports.py
index 01cf3260ae..eb80fded0f 100644
--- a/libs/core/tests/unit_tests/tracers/test_imports.py
+++ b/libs/core/tests/unit_tests/tracers/test_imports.py
@@ -2,13 +2,13 @@ from langchain_core.tracers import __all__
EXPECTED_ALL = [
"BaseTracer",
+ "ConsoleCallbackHandler",
"EvaluatorCallbackHandler",
"LangChainTracer",
- "ConsoleCallbackHandler",
+ "LogStreamCallbackHandler",
"Run",
"RunLog",
"RunLogPatch",
- "LogStreamCallbackHandler",
]