The agent engineering platform.
Python agentsaiai-agentsanthropicchatgptdeepagentsenterpriseframeworkgeminigenerative-ailangchainlanggraphllmmultiagentopen-sourceopenaipydanticpythonragtypescript
feat(core): automatically count and store meta for tool call count (#33756)
Mason Daughertyed9bd6e3adb8
diff --git a/libs/core/langchain_core/tracers/core.py b/libs/core/langchain_core/tracers/core.pyindex 97a2092d15..76a4a0356f 100644--- a/libs/core/langchain_core/tracers/core.py+++ b/libs/core/langchain_core/tracers/core.py@@ -284,6 +284,16 @@ class _TracerCore(ABC):llm_run.end_time = datetime.now(timezone.utc)llm_run.events.append({"name": "end", "time": llm_run.end_time})+ tool_call_count = 0+ for generations in response.generations:+ for generation in generations:+ if hasattr(generation, "message"):+ msg = generation.message+ if hasattr(msg, "tool_calls"):+ tool_call_count += len(msg.tool_calls)+ if tool_call_count > 0:+ llm_run.extra["tool_call_count"] = tool_call_count+return llm_rundef _errored_llm_run(diff --git a/libs/core/tests/unit_tests/tracers/test_automatic_metadata.py b/libs/core/tests/unit_tests/tracers/test_automatic_metadata.pynew file mode 100644index 0000000000..147997c2c1--- /dev/null+++ b/libs/core/tests/unit_tests/tracers/test_automatic_metadata.py@@ -0,0 +1,128 @@+"""Test automatic tool call count storage in tracers."""++from __future__ import annotations++from unittest.mock import MagicMock++from langchain_core.messages import AIMessage+from langchain_core.messages.tool import ToolCall+from langchain_core.outputs import ChatGeneration, LLMResult+from langchain_core.tracers.core import _TracerCore+from langchain_core.tracers.schemas import Run+++class MockTracerCore(_TracerCore):+ """Mock tracer core for testing LLM run completion."""++ def __init__(self) -> None:+ super().__init__()++ def _persist_run(self, run: Run) -> None:+ """Mock implementation of _persist_run."""+++def test_complete_llm_run_automatically_stores_tool_call_count() -> None:+ """Test that `_complete_llm_run` automatically stores tool call count."""+ tracer = MockTracerCore()++ run = MagicMock(spec=Run)+ run.id = "test-llm-run-id"+ run.run_type = "llm"+ run.extra = {}+ run.outputs = {}+ run.events = []+ run.end_time = None+ run.inputs = {}++ tracer.run_map[str(run.id)] = run++ tool_calls = [+ ToolCall(name="search", args={"query": "test"}, id="call_1"),+ ToolCall(name="calculator", args={"expression": "2+2"}, id="call_2"),+ ]+ message = AIMessage(content="Test", tool_calls=tool_calls)+ generation = ChatGeneration(message=message)+ response = LLMResult(generations=[[generation]])++ # Complete the LLM run (this should trigger automatic metadata storage)+ completed_run = tracer._complete_llm_run(response=response, run_id=run.id)++ assert "tool_call_count" in completed_run.extra+ assert completed_run.extra["tool_call_count"] == 2+++def test_complete_llm_run_handles_no_tool_calls() -> None:+ """Test that `_complete_llm_run` handles runs with no tool calls gracefully."""+ tracer = MockTracerCore()++ run = MagicMock(spec=Run)+ run.id = "test-llm-run-id-no-tools"+ run.run_type = "llm"+ run.extra = {}+ run.outputs = {}+ run.events = []+ run.end_time = None+ run.inputs = {}++ tracer.run_map[str(run.id)] = run++ message = AIMessage(content="No tools here")+ generation = ChatGeneration(message=message)+ response = LLMResult(generations=[[generation]])++ completed_run = tracer._complete_llm_run(response=response, run_id=run.id)++ # Verify tool call count is not stored when there are no tool calls+ assert "tool_call_count" not in completed_run.extra+++def test_complete_llm_run_handles_empty_generations() -> None:+ """Test that `_complete_llm_run` handles empty generations gracefully."""+ tracer = MockTracerCore()++ run = MagicMock(spec=Run)+ run.id = "test-llm-run-id-empty"+ run.run_type = "llm"+ run.extra = {}+ run.outputs = {}+ run.events = []+ run.end_time = None+ run.inputs = {}++ tracer.run_map[str(run.id)] = run++ response = LLMResult(generations=[[]])++ completed_run = tracer._complete_llm_run(response=response, run_id=run.id)++ assert "tool_call_count" not in completed_run.extra+++def test_complete_llm_run_counts_tool_calls_from_multiple_generations() -> None:+ """Test that tool calls are counted from multiple generations."""+ tracer = MockTracerCore()++ run = MagicMock(spec=Run)+ run.id = "test-llm-run-id-multi"+ run.run_type = "llm"+ run.extra = {}+ run.outputs = {}+ run.events = []+ run.end_time = None+ run.inputs = {}++ tracer.run_map[str(run.id)] = run++ # Create multiple generations with tool calls+ tool_calls_1 = [ToolCall(name="search", args={"query": "test"}, id="call_1")]+ tool_calls_2 = [+ ToolCall(name="calculator", args={"expression": "2+2"}, id="call_2"),+ ToolCall(name="weather", args={"location": "NYC"}, id="call_3"),+ ]+ gen1 = ChatGeneration(message=AIMessage(content="Gen1", tool_calls=tool_calls_1))+ gen2 = ChatGeneration(message=AIMessage(content="Gen2", tool_calls=tool_calls_2))+ response = LLMResult(generations=[[gen1], [gen2]])++ completed_run = tracer._complete_llm_run(response=response, run_id=run.id)++ assert completed_run.extra["tool_call_count"] == 3diff --git a/libs/core/tests/unit_tests/tracers/test_imports.py b/libs/core/tests/unit_tests/tracers/test_imports.pyindex 01cf3260ae..eb80fded0f 100644--- a/libs/core/tests/unit_tests/tracers/test_imports.py+++ b/libs/core/tests/unit_tests/tracers/test_imports.py@@ -2,13 +2,13 @@ from langchain_core.tracers import __all__EXPECTED_ALL = ["BaseTracer",+ "ConsoleCallbackHandler","EvaluatorCallbackHandler","LangChainTracer",- "ConsoleCallbackHandler",+ "LogStreamCallbackHandler","Run","RunLog","RunLogPatch",- "LogStreamCallbackHandler",]