The agent engineering platform.
Python agentsaiai-agentsanthropicchatgptdeepagentsenterpriseframeworkgeminigenerative-ailangchainlanggraphllmmultiagentopen-sourceopenaipydanticpythonragtypescript
feat(core): add `usage_metadata` to metadata in `LangChainTracer` (#34414)
Hunter Lovell7902fa323876
diff --git a/libs/core/langchain_core/tracers/langchain.py b/libs/core/langchain_core/tracers/langchain.pyindex fdf03de9a4..9fb974ced6 100644--- a/libs/core/langchain_core/tracers/langchain.py+++ b/libs/core/langchain_core/tracers/langchain.py@@ -21,6 +21,7 @@ from typing_extensions import overridefrom langchain_core.env import get_runtime_environmentfrom langchain_core.load import dumpd+from langchain_core.messages.ai import UsageMetadata, add_usagefrom langchain_core.tracers.base import BaseTracerfrom langchain_core.tracers.schemas import Run@@ -69,6 +70,32 @@ def _get_executor() -> ThreadPoolExecutor:return _EXECUTOR+def _get_usage_metadata_from_generations(+ generations: list[list[dict[str, Any]]],+) -> UsageMetadata | None:+ """Extract and aggregate `usage_metadata` from generations.++ Iterates through generations to find and aggregate all `usage_metadata` found in+ messages. This is typically present in chat model outputs.++ Args:+ generations: List of generation batches, where each batch is a list+ of generation dicts that may contain a `'message'` key with+ `'usage_metadata'`.++ Returns:+ The aggregated `usage_metadata` dict if found, otherwise `None`.+ """+ output: UsageMetadata | None = None+ for generation_batch in generations:+ for generation in generation_batch:+ if isinstance(generation, dict) and "message" in generation:+ message = generation["message"]+ if isinstance(message, dict) and "usage_metadata" in message:+ output = add_usage(output, message["usage_metadata"])+ return output++class LangChainTracer(BaseTracer):"""Implementation of the SharedTracer that POSTS to the LangChain endpoint."""@@ -266,6 +293,15 @@ class LangChainTracer(BaseTracer):def _on_llm_end(self, run: Run) -> None:"""Process the LLM Run."""+ # Extract usage_metadata from outputs and store in extra.metadata+ if run.outputs and "generations" in run.outputs:+ usage_metadata = _get_usage_metadata_from_generations(+ run.outputs["generations"]+ )+ if usage_metadata is not None:+ if "metadata" not in run.extra:+ run.extra["metadata"] = {}+ run.extra["metadata"]["usage_metadata"] = usage_metadataself._update_run_single(run)def _on_llm_error(self, run: Run) -> None:diff --git a/libs/core/tests/unit_tests/tracers/test_langchain.py b/libs/core/tests/unit_tests/tracers/test_langchain.pyindex 9d3809877f..011b801b8a 100644--- a/libs/core/tests/unit_tests/tracers/test_langchain.py+++ b/libs/core/tests/unit_tests/tracers/test_langchain.py@@ -12,7 +12,10 @@ from langsmith.run_trees import RunTreefrom langsmith.utils import get_env_var, get_tracer_projectfrom langchain_core.outputs import LLMResult-from langchain_core.tracers.langchain import LangChainTracer+from langchain_core.tracers.langchain import (+ LangChainTracer,+ _get_usage_metadata_from_generations,+)from langchain_core.tracers.schemas import Run@@ -147,6 +150,210 @@ def test_correct_get_tracer_project(assert projects == [expected_project_name]+ ("generations", "expected"),+ [+ # Returns usage_metadata when present+ (+ [+ [+ {+ "text": "Hello!",+ "message": {+ "content": "Hello!",+ "usage_metadata": {+ "input_tokens": 10,+ "output_tokens": 20,+ "total_tokens": 30,+ },+ },+ }+ ]+ ],+ {"input_tokens": 10, "output_tokens": 20, "total_tokens": 30},+ ),+ # Returns None when no usage_metadata+ ([[{"text": "Hello!", "message": {"content": "Hello!"}}]], None),+ # Returns None when no message+ ([[{"text": "Hello!"}]], None),+ # Returns None for empty generations+ ([], None),+ ([[]], None),+ # Aggregates usage_metadata across multiple generations+ (+ [+ [+ {+ "text": "First",+ "message": {+ "content": "First",+ "usage_metadata": {+ "input_tokens": 5,+ "output_tokens": 10,+ "total_tokens": 15,+ },+ },+ },+ {+ "text": "Second",+ "message": {+ "content": "Second",+ "usage_metadata": {+ "input_tokens": 50,+ "output_tokens": 100,+ "total_tokens": 150,+ },+ },+ },+ ]+ ],+ {"input_tokens": 55, "output_tokens": 110, "total_tokens": 165},+ ),+ # Finds usage_metadata across multiple batches+ (+ [+ [{"text": "No message here"}],+ [+ {+ "text": "Has message",+ "message": {+ "content": "Has message",+ "usage_metadata": {+ "input_tokens": 10,+ "output_tokens": 20,+ "total_tokens": 30,+ },+ },+ }+ ],+ ],+ {"input_tokens": 10, "output_tokens": 20, "total_tokens": 30},+ ),+ ],+ ids=[+ "returns_usage_metadata_when_present",+ "returns_none_when_no_usage_metadata",+ "returns_none_when_no_message",+ "returns_none_for_empty_list",+ "returns_none_for_empty_batch",+ "aggregates_across_multiple_generations",+ "finds_across_multiple_batches",+ ],+)+def test_get_usage_metadata_from_generations(+ generations: list[list[dict[str, Any]]], expected: dict[str, Any] | None+) -> None:+ """Test `_get_usage_metadata_from_generations` utility function."""+ result = _get_usage_metadata_from_generations(generations)+ assert result == expected+++def test_on_llm_end_stores_usage_metadata_in_run_extra() -> None:+ """Test that `usage_metadata` is stored in `run.extra.metadata` on llm end."""+ client = unittest.mock.MagicMock(spec=Client)+ client.tracing_queue = None+ tracer = LangChainTracer(client=client)++ run_id = UUID("9d878ab3-e5ca-4218-aef6-44cbdc90160a")+ tracer.on_llm_start({"name": "test_llm"}, ["foo"], run_id=run_id)++ run = tracer.run_map[str(run_id)]+ usage_metadata = {"input_tokens": 100, "output_tokens": 200, "total_tokens": 300}+ run.outputs = {+ "generations": [+ [+ {+ "text": "Hello!",+ "message": {"content": "Hello!", "usage_metadata": usage_metadata},+ }+ ]+ ]+ }++ captured_run = None++ def capture_run(r: Run) -> None:+ nonlocal captured_run+ captured_run = r++ with unittest.mock.patch.object(tracer, "_update_run_single", capture_run):+ tracer._on_llm_end(run)++ assert captured_run is not None+ assert "metadata" in captured_run.extra+ assert captured_run.extra["metadata"]["usage_metadata"] == usage_metadata+++def test_on_llm_end_no_usage_metadata_when_not_present() -> None:+ """Test that no `usage_metadata` is added when not present in outputs."""+ client = unittest.mock.MagicMock(spec=Client)+ client.tracing_queue = None+ tracer = LangChainTracer(client=client)++ run_id = UUID("9d878ab3-e5ca-4218-aef6-44cbdc90160a")+ tracer.on_llm_start({"name": "test_llm"}, ["foo"], run_id=run_id)++ run = tracer.run_map[str(run_id)]+ run.outputs = {+ "generations": [[{"text": "Hello!", "message": {"content": "Hello!"}}]]+ }++ captured_run = None++ def capture_run(r: Run) -> None:+ nonlocal captured_run+ captured_run = r++ with unittest.mock.patch.object(tracer, "_update_run_single", capture_run):+ tracer._on_llm_end(run)++ assert captured_run is not None+ extra_metadata = captured_run.extra.get("metadata", {})+ assert "usage_metadata" not in extra_metadata+++def test_on_llm_end_preserves_existing_metadata() -> None:+ """Test that existing metadata is preserved when adding `usage_metadata`."""+ client = unittest.mock.MagicMock(spec=Client)+ client.tracing_queue = None+ tracer = LangChainTracer(client=client)++ run_id = UUID("9d878ab3-e5ca-4218-aef6-44cbdc90160a")+ tracer.on_llm_start(+ {"name": "test_llm"},+ ["foo"],+ run_id=run_id,+ metadata={"existing_key": "existing_value"},+ )++ run = tracer.run_map[str(run_id)]+ usage_metadata = {"input_tokens": 10, "output_tokens": 20, "total_tokens": 30}+ run.outputs = {+ "generations": [+ [+ {+ "text": "Hello!",+ "message": {"content": "Hello!", "usage_metadata": usage_metadata},+ }+ ]+ ]+ }++ captured_run = None++ def capture_run(r: Run) -> None:+ nonlocal captured_run+ captured_run = r++ with unittest.mock.patch.object(tracer, "_update_run_single", capture_run):+ tracer._on_llm_end(run)++ assert captured_run is not None+ assert "metadata" in captured_run.extra+ assert captured_run.extra["metadata"]["usage_metadata"] == usage_metadata+ assert captured_run.extra["metadata"]["existing_key"] == "existing_value"++def test_on_chain_start_skips_persist_when_defers_inputs() -> None:"""Test that `_on_chain_start` skips persist when `defers_inputs` is set."""client = unittest.mock.MagicMock(spec=Client)