Skip to content

The agent engineering platform.

feat(core): Add chat model and LLM invocation params to traceable metadata (#36771)

Jacob Leec04e05feb14b
diff --git a/libs/core/langchain_core/language_models/_utils.py b/libs/core/langchain_core/language_models/_utils.py
index c134e7cd84..289b675307 100644
--- a/libs/core/langchain_core/language_models/_utils.py
+++ b/libs/core/langchain_core/language_models/_utils.py
@@ -2,6 +2,7 @@ import re
from collections.abc import Sequence
from typing import (
TYPE_CHECKING,
+ Any,
Literal,
TypedDict,
TypeVar,
@@ -14,6 +15,21 @@ from langchain_core.messages.content import (
)
+def _filter_invocation_params_for_tracing(params: dict[str, Any]) -> dict[str, Any]:
+ """Filter out large/inappropriate fields from invocation params for tracing.
+
+ Removes fields like tools, functions, messages, response_format that can be large.
+
+ Args:
+ params: The invocation parameters to filter.
+
+ Returns:
+ The filtered parameters with large fields removed.
+ """
+ excluded_keys = {"tools", "functions", "messages", "response_format"}
+ return {k: v for k, v in params.items() if k not in excluded_keys}
+
+
def is_openai_data_block(
block: dict, filter_: Literal["image", "audio", "file"] | None = None
) -> bool:
diff --git a/libs/core/langchain_core/language_models/chat_models.py b/libs/core/langchain_core/language_models/chat_models.py
index f30b841996..5cbb92547b 100644
--- a/libs/core/langchain_core/language_models/chat_models.py
+++ b/libs/core/langchain_core/language_models/chat_models.py
@@ -25,6 +25,7 @@ from langchain_core.callbacks import (
)
from langchain_core.globals import get_llm_cache
from langchain_core.language_models._utils import (
+ _filter_invocation_params_for_tracing,
_normalize_messages,
_update_message_content_to_blocks,
)
@@ -567,6 +568,9 @@ class BaseChatModel(BaseLanguageModel[AIMessage], ABC):
self.tags,
inheritable_metadata,
self.metadata,
+ langsmith_inheritable_metadata=_filter_invocation_params_for_tracing(
+ params
+ ),
)
(run_manager,) = callback_manager.on_chat_model_start(
self._serialized,
@@ -695,6 +699,9 @@ class BaseChatModel(BaseLanguageModel[AIMessage], ABC):
self.tags,
inheritable_metadata,
self.metadata,
+ langsmith_inheritable_metadata=_filter_invocation_params_for_tracing(
+ params
+ ),
)
(run_manager,) = await callback_manager.on_chat_model_start(
self._serialized,
@@ -972,6 +979,9 @@ class BaseChatModel(BaseLanguageModel[AIMessage], ABC):
self.tags,
inheritable_metadata,
self.metadata,
+ langsmith_inheritable_metadata=_filter_invocation_params_for_tracing(
+ params
+ ),
)
messages_to_trace = [
_format_for_tracing(message_list) for message_list in messages
@@ -1095,6 +1105,9 @@ class BaseChatModel(BaseLanguageModel[AIMessage], ABC):
self.tags,
inheritable_metadata,
self.metadata,
+ langsmith_inheritable_metadata=_filter_invocation_params_for_tracing(
+ params
+ ),
)
messages_to_trace = [
diff --git a/libs/core/langchain_core/language_models/llms.py b/libs/core/langchain_core/language_models/llms.py
index e5400d16d4..1ace9cb554 100644
--- a/libs/core/langchain_core/language_models/llms.py
+++ b/libs/core/langchain_core/language_models/llms.py
@@ -42,6 +42,7 @@ from langchain_core.callbacks import (
Callbacks,
)
from langchain_core.globals import get_llm_cache
+from langchain_core.language_models._utils import _filter_invocation_params_for_tracing
from langchain_core.language_models.base import (
BaseLanguageModel,
LangSmithParams,
@@ -537,6 +538,9 @@ class BaseLLM(BaseLanguageModel[str], ABC):
self.tags,
inheritable_metadata,
self.metadata,
+ langsmith_inheritable_metadata=_filter_invocation_params_for_tracing(
+ params
+ ),
)
(run_manager,) = callback_manager.on_llm_start(
self._serialized,
@@ -607,6 +611,9 @@ class BaseLLM(BaseLanguageModel[str], ABC):
self.tags,
inheritable_metadata,
self.metadata,
+ langsmith_inheritable_metadata=_filter_invocation_params_for_tracing(
+ params
+ ),
)
(run_manager,) = await callback_manager.on_llm_start(
self._serialized,
@@ -950,6 +957,8 @@ class BaseLLM(BaseLanguageModel[str], ABC):
run_name_list = run_name or cast(
"list[str | None]", ([None] * len(prompts))
)
+ params = self.dict()
+ params["stop"] = stop
callback_managers = [
CallbackManager.configure(
callback,
@@ -959,6 +968,9 @@ class BaseLLM(BaseLanguageModel[str], ABC):
self.tags,
meta,
self.metadata,
+ langsmith_inheritable_metadata=_filter_invocation_params_for_tracing(
+ params
+ ),
)
for callback, tag, meta in zip(
callbacks, tags_list, metadata_list, strict=False
@@ -966,6 +978,8 @@ class BaseLLM(BaseLanguageModel[str], ABC):
]
else:
# We've received a single callbacks arg to apply to all inputs
+ params = self.dict()
+ params["stop"] = stop
callback_managers = [
CallbackManager.configure(
cast("Callbacks", callbacks),
@@ -975,12 +989,13 @@ class BaseLLM(BaseLanguageModel[str], ABC):
self.tags,
cast("dict[str, Any]", metadata),
self.metadata,
+ langsmith_inheritable_metadata=_filter_invocation_params_for_tracing(
+ params
+ ),
)
] * len(prompts)
run_name_list = [cast("str | None", run_name)] * len(prompts)
run_ids_list = self._get_run_ids_list(run_id, prompts)
- params = self.dict()
- params["stop"] = stop
options = {"stop": stop}
(
existing_prompts,
@@ -1214,6 +1229,8 @@ class BaseLLM(BaseLanguageModel[str], ABC):
run_name_list = run_name or cast(
"list[str | None]", ([None] * len(prompts))
)
+ params = self.dict()
+ params["stop"] = stop
callback_managers = [
AsyncCallbackManager.configure(
callback,
@@ -1223,6 +1240,9 @@ class BaseLLM(BaseLanguageModel[str], ABC):
self.tags,
meta,
self.metadata,
+ langsmith_inheritable_metadata=_filter_invocation_params_for_tracing(
+ params
+ ),
)
for callback, tag, meta in zip(
callbacks, tags_list, metadata_list, strict=False
@@ -1230,6 +1250,8 @@ class BaseLLM(BaseLanguageModel[str], ABC):
]
else:
# We've received a single callbacks arg to apply to all inputs
+ params = self.dict()
+ params["stop"] = stop
callback_managers = [
AsyncCallbackManager.configure(
cast("Callbacks", callbacks),
@@ -1239,12 +1261,13 @@ class BaseLLM(BaseLanguageModel[str], ABC):
self.tags,
cast("dict[str, Any]", metadata),
self.metadata,
+ langsmith_inheritable_metadata=_filter_invocation_params_for_tracing(
+ params
+ ),
)
] * len(prompts)
run_name_list = [cast("str | None", run_name)] * len(prompts)
run_ids_list = self._get_run_ids_list(run_id, prompts)
- params = self.dict()
- params["stop"] = stop
options = {"stop": stop}
(
existing_prompts,
diff --git a/libs/core/tests/unit_tests/language_models/chat_models/test_base.py b/libs/core/tests/unit_tests/language_models/chat_models/test_base.py
index e2475ef3d3..4b7206de46 100644
--- a/libs/core/tests/unit_tests/language_models/chat_models/test_base.py
+++ b/libs/core/tests/unit_tests/language_models/chat_models/test_base.py
@@ -3,7 +3,9 @@
import uuid
import warnings
from collections.abc import AsyncIterator, Iterator
+from contextlib import contextmanager
from typing import TYPE_CHECKING, Any, Literal
+from unittest.mock import patch
import pytest
from pydantic import model_validator
@@ -17,8 +19,14 @@ from langchain_core.language_models import (
FakeListChatModel,
ParrotFakeChatModel,
)
-from langchain_core.language_models._utils import _normalize_messages
-from langchain_core.language_models.chat_models import _generate_response_from_error
+from langchain_core.language_models._utils import (
+ _filter_invocation_params_for_tracing,
+ _normalize_messages,
+)
+from langchain_core.language_models.chat_models import (
+ SimpleChatModel,
+ _generate_response_from_error,
+)
from langchain_core.language_models.fake_chat_models import (
FakeListChatModelError,
GenericFakeChatModel,
@@ -37,6 +45,7 @@ from langchain_core.tracers import LogStreamCallbackHandler
from langchain_core.tracers.base import BaseTracer
from langchain_core.tracers.context import collect_runs
from langchain_core.tracers.event_stream import _AstreamEventsCallbackHandler
+from langchain_core.tracers.langchain import LangChainTracer
from langchain_core.tracers.schemas import Run
from tests.unit_tests.fake.callbacks import (
BaseFakeCallbackHandler,
@@ -321,6 +330,20 @@ class FakeTracer(BaseTracer):
self.traced_run_ids.append(run.id)
+class LangChainTracerRunCollector:
+ def __init__(self) -> None:
+ self.tracer = LangChainTracer()
+ self.runs: list[Run] = []
+
+ @contextmanager
+ def tracing_callback(self) -> Iterator[LangChainTracer]:
+ def collect_tracer_run(_: LangChainTracer, run: Run) -> None:
+ self.runs.append(run)
+
+ with patch.object(LangChainTracer, "_persist_run", new=collect_tracer_run):
+ yield self.tracer
+
+
def test_pass_run_id() -> None:
llm = FakeListChatModel(responses=["a", "b", "c"])
cb = FakeTracer()
@@ -1390,3 +1413,100 @@ def test_generate_response_from_error_handles_streaming_response_failure() -> No
assert metadata["body"] is None
assert metadata["headers"] == {"content-type": "application/json"}
assert metadata["status_code"] == 400
+
+
+def test_filter_invocation_params_for_tracing() -> None:
+ """Test that large fields are filtered from invocation params for tracing."""
+ params = {
+ "temperature": 0.7,
+ "tools": [{"name": "test_tool"}],
+ "functions": [{"name": "test_function"}],
+ "messages": [{"role": "system", "content": "test"}],
+ "response_format": {"type": "json_object"},
+ }
+ filtered = _filter_invocation_params_for_tracing(params)
+
+ # Should include temperature
+ assert "temperature" in filtered
+ assert filtered["temperature"] == 0.7
+
+ # Should exclude these large fields
+ assert "tools" not in filtered
+ assert "functions" not in filtered
+ assert "messages" not in filtered
+ assert "response_format" not in filtered
+
+
+class FakeChatModelWithInvocationParams(SimpleChatModel):
+ """Fake chat model with invocation params for testing tracing."""
+
+ temperature: float = 0.7
+
+ @property
+ @override
+ def _llm_type(self) -> str:
+ return "fake-chat-model-with-invocation-params"
+
+ @property
+ @override
+ def _identifying_params(self) -> dict[str, Any]:
+ return {
+ "temperature": self.temperature,
+ "tools": [{"name": "test_tool"}],
+ "functions": [{"name": "test_function"}],
+ "messages": [{"role": "system", "content": "test"}],
+ "response_format": {"type": "json_object"},
+ }
+
+ @override
+ def _call(
+ self,
+ messages: list[BaseMessage],
+ stop: list[str] | None = None,
+ run_manager: CallbackManagerForLLMRun | None = None,
+ **kwargs: Any,
+ ) -> str:
+ return "test response"
+
+
+def test_invocation_params_passed_to_tracer_metadata() -> None:
+ """Test that invocation params are passed to tracer metadata."""
+ llm = FakeChatModelWithInvocationParams()
+ collector = LangChainTracerRunCollector()
+
+ with collector.tracing_callback() as tracer:
+ llm.invoke([HumanMessage(content="Hello")], config={"callbacks": [tracer]})
+
+ assert len(collector.runs) == 1
+ run = collector.runs[0]
+
+ key = "LANGSMITH_LANGGRAPH_API_VARIANT"
+
+ if key in run.extra["metadata"]:
+ del run.extra["metadata"][key]
+
+ assert run.extra == {
+ "batch_size": 1,
+ "invocation_params": {
+ "_type": "fake-chat-model-with-invocation-params",
+ "functions": [{"name": "test_function"}],
+ "messages": [{"content": "test", "role": "system"}],
+ "response_format": {"type": "json_object"},
+ "stop": None,
+ "temperature": 0.7,
+ "tools": [{"name": "test_tool"}],
+ },
+ "metadata": {
+ "_type": "fake-chat-model-with-invocation-params",
+ "ls_integration": "langchain_chat_model",
+ "ls_model_type": "chat",
+ "ls_provider": "fakechatmodelwithinvocationparams",
+ "ls_temperature": 0.7,
+ "revision_id": run.extra["metadata"]["revision_id"],
+ "stop": None,
+ "temperature": 0.7,
+ },
+ "options": {"stop": None},
+ "runtime": run.extra["runtime"],
+ }
+ assert run.metadata == run.extra["metadata"]
diff --git a/libs/core/tests/unit_tests/language_models/llms/test_base.py b/libs/core/tests/unit_tests/language_models/llms/test_base.py
index 7ca52e4339..5b99cbdd26 100644
--- a/libs/core/tests/unit_tests/language_models/llms/test_base.py
+++ b/libs/core/tests/unit_tests/language_models/llms/test_base.py
@@ -13,6 +13,7 @@ from langchain_core.language_models import (
BaseLLM,
FakeListLLM,
)
+from langchain_core.language_models._utils import _filter_invocation_params_for_tracing
from langchain_core.outputs import Generation, GenerationChunk, LLMResult
from langchain_core.tracers.context import collect_runs
from tests.unit_tests.fake.callbacks import (
@@ -284,3 +285,94 @@ def test_get_ls_params() -> None:
ls_params = llm._get_ls_params(stop=["stop"])
assert ls_params["ls_stop"] == ["stop"]
+
+
+def test_filter_invocation_params_for_tracing() -> None:
+ """Test that large fields are filtered from invocation params for tracing."""
+ params = {
+ "temperature": 0.7,
+ "tools": [{"name": "test_tool"}],
+ "functions": [{"name": "test_function"}],
+ "messages": [{"role": "system", "content": "test"}],
+ "response_format": {"type": "json_object"},
+ }
+ filtered = _filter_invocation_params_for_tracing(params)
+
+ # Should include temperature
+ assert "temperature" in filtered
+ assert filtered["temperature"] == 0.7
+
+ # Should exclude these large fields
+ assert "tools" not in filtered
+ assert "functions" not in filtered
+ assert "messages" not in filtered
+ assert "response_format" not in filtered
+
+
+class FakeLLMWithInvocationParams(BaseLLM):
+ """Fake LLM with invocation params for testing tracing."""
+
+ temperature: float = 0.7
+
+ @property
+ @override
+ def _llm_type(self) -> str:
+ return "fake-llm-with-invocation-params"
+
+ @property
+ @override
+ def _identifying_params(self) -> dict[str, Any]:
+ return {
+ "temperature": self.temperature,
+ "tools": [{"name": "test_tool"}],
+ "functions": [{"name": "test_function"}],
+ "messages": [{"role": "system", "content": "test"}],
+ "response_format": {"type": "json_object"},
+ }
+
+ @override
+ def _generate(
+ self,
+ prompts: list[str],
+ stop: list[str] | None = None,
+ run_manager: CallbackManagerForLLMRun | None = None,
+ **kwargs: Any,
+ ) -> LLMResult:
+ generations = [[Generation(text="test response")]]
+ return LLMResult(generations=generations)
+
+ @override
+ async def _agenerate(
+ self,
+ prompts: list[str],
+ stop: list[str] | None = None,
+ run_manager: AsyncCallbackManagerForLLMRun | None = None,
+ **kwargs: Any,
+ ) -> LLMResult:
+ generations = [[Generation(text="test response")]]
+ return LLMResult(generations=generations)
+
+
+async def test_llm_invocation_params_filtered_in_stream() -> None:
+ """Test that invocation params are filtered when streaming."""
+
+ # Create a custom LLM that supports streaming
+ class FakeStreamingLLM(FakeLLMWithInvocationParams):
+ @override
+ def _stream(
+ self,
+ prompt: str,
+ stop: list[str] | None = None,
+ run_manager: CallbackManagerForLLMRun | None = None,
+ **kwargs: Any,
+ ) -> Iterator[GenerationChunk]:
+ yield GenerationChunk(text="test ")
+
+ streaming_llm = FakeStreamingLLM()
+
+ with collect_runs() as cb:
+ list(streaming_llm.stream("Hello", config={"callbacks": [cb]}))
+ assert len(cb.traced_runs) == 1
+ run = cb.traced_runs[0]
+ # Verify the run was traced
+ assert run.extra is not None
diff --git a/libs/core/uv.lock b/libs/core/uv.lock
index 9dab8c14a3..49e83e09e8 100644
--- a/libs/core/uv.lock
+++ b/libs/core/uv.lock
@@ -1134,7 +1134,7 @@ typing = [
[[package]]
name = "langchain-text-splitters"
-version = "1.1.1"
+version = "1.1.2"
source = { directory = "../text-splitters" }
dependencies = [
{ name = "langchain-core" },