The agent engineering platform.
Python agentsaiai-agentsanthropicchatgptdeepagentsenterpriseframeworkgeminigenerative-ailangchainlanggraphllmmultiagentopen-sourceopenaipydanticpythonragtypescript
feat(langchain): accept `summarizer` in `SummarizationMiddleware` (#41225)
ccurmea0074fb7e5f3
diff --git a/libs/langchain_v1/langchain/agents/middleware/summarization.py b/libs/langchain_v1/langchain/agents/middleware/summarization.pyindex 585de9ba95..a63857c139 100644--- a/libs/langchain_v1/langchain/agents/middleware/summarization.py+++ b/libs/langchain_v1/langchain/agents/middleware/summarization.py@@ -3,7 +3,7 @@import loggingimport uuidimport warnings-from collections.abc import Callable, Iterable, Mapping+from collections.abc import Callable, Coroutine, Iterable, Mappingfrom functools import partialfrom typing import Any, Literal, TypedDict, cast@@ -21,7 +21,8 @@ from langchain_core.messages.utils import (get_buffer_string,trim_messages,)-from langchain_core.runnables import RunnableConfig+from langchain_core.runnables import Runnable, RunnableConfig+from langchain_core.runnables.base import coerce_to_runnablefrom langgraph.graph.message import (REMOVE_ALL_MESSAGES,)@@ -40,6 +41,19 @@ logger = logging.getLogger(__name__)TokenCounter = Callable[[Iterable[MessageLikeRepresentation]], int]+Summarizer = (+ str+ | BaseChatModel+ | Runnable[list[AnyMessage], str]+ | Callable[[list[AnyMessage]], str]+ | Callable[[list[AnyMessage]], Coroutine[Any, Any, str]]+)+"""Generates a summary from the messages selected for summarization.++Either a chat model (or model string) prompted with the summary prompt, or a+`Runnable` or callable taking the messages and returning the summary text.+"""+DEFAULT_SUMMARY_PROMPT = """<role>Context Extraction Assistant</role>@@ -121,6 +135,14 @@ def _summary_config() -> RunnableConfig:return {"metadata": {"lc_source": "summarization", **internal_call_metadata()}}+def _check_summary(summary: object) -> str:+ """Validate and normalize the text returned by a custom `summarizer`."""+ if not isinstance(summary, str):+ msg = f"`summarizer` must return a str, got {type(summary).__name__}."+ raise TypeError(msg)+ return summary.strip()++def _provider_matches(message_provider: str, model_ls_provider: str | None) -> bool:if model_ls_provider is None:return False@@ -250,6 +272,10 @@ class SummarizationMiddleware(AgentMiddleware[AgentState[ResponseT], ContextT, Rthe model's context window (`ContextOverflowError`), the call is retried once withthe middle of the history cut out. If a summary call still fails, theunderlying error propagates rather than fabricating a summary.++ To generate summaries with something other than a chat model, pass a+ `summarizer`. The middleware still decides when to summarize and which+ messages to summarize; only the summary text comes from the summarizer."""transformers = (InternalCallTransformer,)@@ -268,6 +294,7 @@ class SummarizationMiddleware(AgentMiddleware[AgentState[ResponseT], ContextT, Rtoken_counter: TokenCounter = count_tokens_approximately,summary_prompt: str = DEFAULT_SUMMARY_PROMPT,trim_tokens_to_summarize: int | None = _DEFAULT_TRIM_TOKEN_LIMIT,+ summarizer: Summarizer | None = None,**deprecated_kwargs: Any,) -> None:"""Initialize summarization middleware.@@ -336,6 +363,47 @@ class SummarizationMiddleware(AgentMiddleware[AgentState[ResponseT], ContextT, Rthe summarization call.Pass `None` to skip trimming entirely.+ summarizer: Optional replacement for the summarization model call.++ `model` is still used for its profile (fractional limits) and+ token counting, but is not called to write summaries.++ - A chat model, or a model string resolved with+ `init_chat_model`, writes summaries in place of `model`. It is+ prompted with `summary_prompt` and retried like `model` would+ be. Use this to summarize with a different model than the one+ whose context window you're managing. To summarize with a chat+ model, pass its name or a `BaseChatModel` instance; wrapped+ models such as `model.with_config(...)` are treated as a+ `Runnable` (below).+ - A `Runnable` or function receives the messages to summarize+ (after trimming) and returns the summary text. `summary_prompt`+ is not used. Functions may be sync or async; an async function+ requires async invocation (e.g. `ainvoke`), and a sync function+ runs in a thread pool when invoked asynchronously.++ !!! example++ ```python+ # Manage gpt-6-sol's context window, summarize with gpt-6-luna+ SummarizationMiddleware(+ model="openai:gpt-6-sol",+ trigger=("fraction", 0.8),+ summarizer="openai:gpt-6-luna",+ )+++ # Summarize with a non-LLM service+ async def summarize(messages: Sequence[BaseMessage]) -> str:+ return await my_summarization_service(get_buffer_string(messages))+++ SummarizationMiddleware(+ model="openai:gpt-6-luna",+ trigger=("fraction", 0.8),+ summarizer=summarize,+ )+ ```"""# Handle deprecated parametersif "max_tokens_before_summary" in deprecated_kwargs:@@ -389,6 +457,16 @@ class SummarizationMiddleware(AgentMiddleware[AgentState[ResponseT], ContextT, Rself._partial_token_counter = token_counterself.summary_prompt = summary_promptself.trim_tokens_to_summarize = trim_tokens_to_summarize+ if isinstance(summarizer, str):+ summarizer = init_chat_model(summarizer)+ # The chat model that writes summaries: `model` unless `summarizer` replaces it.+ self._summary_chat_model = self.model+ self.summarizer: Runnable[list[AnyMessage], str] | None = None+ if isinstance(summarizer, BaseChatModel):+ self._summary_chat_model = summarizer+ self._summary_model = summarizer.with_retry(retry_if_exception_type=default_retry_on)+ elif summarizer is not None:+ self.summarizer = coerce_to_runnable(summarizer)requires_profile = any("fraction" in clause for clause in self._trigger_clauses)if self.keep[0] == "fraction":@@ -726,10 +804,10 @@ class SummarizationMiddleware(AgentMiddleware[AgentState[ResponseT], ContextT, R# Advance past any ToolMessages to avoid splitting AI/Tool pairsreturn self._find_safe_cutoff_point(messages, cutoff_candidate)- def _get_profile_limits(self) -> int | None:- """Retrieve max input token limit from the model profile."""+ def _get_profile_limits(self, model: BaseChatModel | None = None) -> int | None:+ """Retrieve max input token limit from the profile of `model` (default: `self.model`)."""try:- profile = self.model.profile+ profile = (model or self.model).profileexcept AttributeError:return None@@ -858,6 +936,11 @@ class SummarizationMiddleware(AgentMiddleware[AgentState[ResponseT], ContextT, Rif not trimmed_messages:return "Previous conversation was too long to summarize."+ if self.summarizer is not None:+ return _check_summary(+ self.summarizer.invoke(trimmed_messages, config=_summary_config())+ )+prompt = self._format_summary_prompt(trimmed_messages)try:response = self._summary_model.invoke(prompt, config=_summary_config())@@ -888,6 +971,10 @@ class SummarizationMiddleware(AgentMiddleware[AgentState[ResponseT], ContextT, Rif not trimmed_messages:return "Previous conversation was too long to summarize."+ if self.summarizer is not None:+ summary = await self.summarizer.ainvoke(trimmed_messages, config=_summary_config())+ return _check_summary(summary)+prompt = self._format_summary_prompt(trimmed_messages)try:response = await self._summary_model.ainvoke(prompt, config=_summary_config())@@ -910,7 +997,7 @@ class SummarizationMiddleware(AgentMiddleware[AgentState[ResponseT], ContextT, RCuts the middle out of the formatted history, keeping its start (where theobjective is usually stated) and its most recent part, within half of its- token count and 40% of the model's `max_input_tokens` when known.+ token count and 40% of the summary model's `max_input_tokens` when known.Returns:The retry prompt, or `None` if nothing fits the budget.@@ -918,7 +1005,7 @@ class SummarizationMiddleware(AgentMiddleware[AgentState[ResponseT], ContextT, Rformatted = get_buffer_string(messages, format="xml")tokens = self._partial_token_counter([HumanMessage(content=formatted)])budget = tokens // 2- if (max_input_tokens := self._get_profile_limits()) is not None:+ if (max_input_tokens := self._get_profile_limits(self._summary_chat_model)) is not None:# 40% rather than half absorbs up to 1.25x usage-metadata scaling.budget = min(budget, max_input_tokens * 2 // 5)keep_chars = len(formatted) * budget // max(tokens, 1)diff --git a/libs/langchain_v1/tests/unit_tests/agents/middleware/implementations/test_summarization.py b/libs/langchain_v1/tests/unit_tests/agents/middleware/implementations/test_summarization.pyindex 7315ba6a0d..acf00efffa 100644--- a/libs/langchain_v1/tests/unit_tests/agents/middleware/implementations/test_summarization.py+++ b/libs/langchain_v1/tests/unit_tests/agents/middleware/implementations/test_summarization.py@@ -24,7 +24,7 @@ from langchain_core.messages import ()from langchain_core.messages.utils import count_tokens_approximately, get_buffer_stringfrom langchain_core.outputs import ChatGeneration, ChatResult-from langchain_core.runnables import RunnableConfig+from langchain_core.runnables import RunnableConfig, RunnableLambdafrom langgraph.checkpoint.memory import InMemorySaverfrom langgraph.graph.message import REMOVE_ALL_MESSAGESfrom langgraph.runtime import Runtime@@ -2473,3 +2473,269 @@ class TestSummarizationStreamingEndToEnd:texts.append(text)assert texts == ["Final answer."]+++def test_summarizer_replaces_model_call() -> None:+ """A `summarizer` writes the summary; the model and `summary_prompt` are not used."""+ received: list[list[AnyMessage]] = []++ def summarizer(messages: list[AnyMessage]) -> str:+ received.append(messages)+ return " Custom summary "++ middleware = SummarizationMiddleware(+ model=_AlwaysFailingModel(),+ trigger=("messages", 3),+ keep=("messages", 1),+ summary_prompt="unused {messages}",+ summarizer=summarizer,+ )+ messages: list[AnyMessage] = [+ HumanMessage(content="hi", id="h0"),+ AIMessage(content="hello", id="a0"),+ HumanMessage(content="how are you", id="h1"),+ ]++ result = middleware.before_model(AgentState[Any](messages=messages), Runtime())++ assert result is not None+ assert received == [messages[:2]]+ assert isinstance(result["messages"][1], HumanMessage)+ assert result["messages"][1].text.endswith("Custom summary")+ assert result["messages"][2:] == messages[2:]+++def test_summarizer_receives_trimmed_messages() -> None:+ """The summarizer sees the messages after `trim_tokens_to_summarize` is applied."""+ received: list[list[AnyMessage]] = []++ def summarizer(messages: list[AnyMessage]) -> str:+ received.append(messages)+ return "summary"++ middleware = SummarizationMiddleware(+ model=MockChatModel(), trim_tokens_to_summarize=None, summarizer=summarizer+ )+ messages: list[AnyMessage] = [HumanMessage(content=str(i)) for i in range(5)]+ with patch.object(middleware, "_trim_messages_for_summary", return_value=messages[-2:]) as trim:+ assert middleware._create_summary(messages) == "summary"++ trim.assert_called_once_with(messages)+ assert received == [messages[-2:]]+++def test_summarizer_accepts_runnable_and_gets_summarization_metadata() -> None:+ """A `Runnable` summarizer is invoked with the summarization metadata."""+ configs: list[RunnableConfig] = []++ def summarize(messages: list[AnyMessage], config: RunnableConfig) -> str:+ configs.append(config)+ return get_buffer_string(messages)++ middleware = SummarizationMiddleware(+ model=_AlwaysFailingModel(), summarizer=RunnableLambda(summarize)+ )++ assert middleware._create_summary([HumanMessage(content="hi")]) == "Human: hi"+ metadata = configs[0]["metadata"]+ assert metadata["lc_source"] == "summarization"+ assert (+ metadata[INTERNAL_CALL_METADATA_KEY] == internal_call_metadata()[INTERNAL_CALL_METADATA_KEY]+ )+++async def test_summarizer_async() -> None:+ """Async invocation uses the summarizer's async implementation."""++ async def summarize(messages: list[AnyMessage]) -> str:+ return f"{len(messages)} messages"++ middleware = SummarizationMiddleware(+ model=_AlwaysFailingModel(),+ trigger=("messages", 3),+ keep=("messages", 1),+ summarizer=summarize,+ )+ messages: list[AnyMessage] = [+ HumanMessage(content="hi", id="h0"),+ AIMessage(content="hello", id="a0"),+ HumanMessage(content="how are you", id="h1"),+ ]++ result = await middleware.abefore_model(AgentState[Any](messages=messages), Runtime())++ assert result is not None+ assert result["messages"][1].text.endswith("2 messages")+++def test_summarizer_errors_propagate() -> None:+ """Errors raised by the summarizer propagate and leave history untouched."""++ def summarizer(_: list[AnyMessage]) -> str:+ msg = "summarizer down"+ raise RuntimeError(msg)++ middleware = SummarizationMiddleware(+ model=MockChatModel(),+ trigger=("messages", 2),+ keep=("messages", 1),+ summarizer=summarizer,+ )+ messages: list[AnyMessage] = [+ HumanMessage(content="hi", id="h0"),+ AIMessage(content="hello", id="a0"),+ ]+ state = AgentState[Any](messages=messages)++ with pytest.raises(RuntimeError, match="summarizer down"):+ middleware.before_model(state, Runtime())+ assert state["messages"] == messages+++def test_summarizer_empty_history_skips_summarizer() -> None:+ """The built-in empty-history message is returned without calling the summarizer."""++ def summarizer(_: list[AnyMessage]) -> str:+ msg = "should not be called"+ raise AssertionError(msg)++ middleware = SummarizationMiddleware(model=MockChatModel(), summarizer=summarizer)++ assert middleware._create_summary([]) == "No previous conversation history."+++def test_chat_model_summarizer_uses_summary_prompt() -> None:+ """A chat model `summarizer` writes summaries with `summary_prompt`; `model` isn't called."""+ summary_model = _OverflowingModel(max_chars=1_000_000)+ middleware = SummarizationMiddleware(+ model=_AlwaysFailingModel(),+ summary_prompt="Summarize this: {messages}",+ summarizer=summary_model,+ )++ assert middleware._create_summary([HumanMessage(content="hi")]) == "Summary."+ assert len(summary_model.prompts) == 1+ assert summary_model.prompts[0].startswith("Summarize this: ")+ assert "hi" in summary_model.prompts[0]+ assert middleware.summarizer is None+++async def test_chat_model_summarizer_async() -> None:+ summary_model = _OverflowingModel(max_chars=1_000_000)+ middleware = SummarizationMiddleware(model=_AlwaysFailingModel(), summarizer=summary_model)++ assert await middleware._acreate_summary([HumanMessage(content="hi")]) == "Summary."+ assert len(summary_model.prompts) == 1+++def test_chat_model_summarizer_keeps_profile_of_model() -> None:+ """Fractional limits still come from `model`'s profile, not the summarizer's."""+ summary_model = _OverflowingModel(max_chars=1_000_000)+ middleware = SummarizationMiddleware(+ model=ProfileChatModel(),+ trigger=("fraction", 0.5),+ summarizer=summary_model,+ )++ assert middleware._get_profile_limits() == 1000+ assert middleware.model is not summary_model+++def test_string_summarizer_is_resolved_with_init_chat_model() -> None:+ summary_model = _OverflowingModel(max_chars=1_000_000)+ with patch(+ "langchain.agents.middleware.summarization.init_chat_model",+ return_value=summary_model,+ ) as init:+ middleware = SummarizationMiddleware(+ model=_AlwaysFailingModel(), summarizer="openai:gpt-5-mini"+ )++ init.assert_called_once_with("openai:gpt-5-mini")+ assert middleware._create_summary([HumanMessage(content="hi")]) == "Summary."+++def test_chat_model_summarizer_does_not_retry_non_retryable_errors() -> None:+ class AuthFailingModel(BaseChatModel):+ attempts: int = 0++ @override+ def _generate(+ self,+ messages: list[BaseMessage],+ stop: list[str] | None = None,+ run_manager: CallbackManagerForLLMRun | None = None,+ **kwargs: Any,+ ) -> ChatResult:+ self.attempts += 1+ msg = "invalid api key"+ raise ModelAuthenticationError(msg)++ @property+ def _llm_type(self) -> str:+ return "mock"++ summary_model = AuthFailingModel()+ middleware = SummarizationMiddleware(model=MockChatModel(), summarizer=summary_model)+ middleware._summary_model.wait_exponential_jitter = False # type: ignore[attr-defined]++ with pytest.raises(ModelAuthenticationError):+ middleware._create_summary([HumanMessage(content="hi")])+ assert summary_model.attempts == 1+++def test_chat_model_summarizer_overflow_budget_uses_summarizer_profile() -> None:+ """The overflow retry is sized to the summary model's context window."""+ uncapped = _OverflowingModel(max_chars=25_000)+ SummarizationMiddleware(+ model=MockChatModel(), trim_tokens_to_summarize=None, summarizer=uncapped+ )._create_summary(_long_history())++ capped = _OverflowingModel(max_chars=25_000, profile=ModelProfile(max_input_tokens=1000))+ middleware = SummarizationMiddleware(+ model=MockChatModel(), trim_tokens_to_summarize=None, summarizer=capped+ )++ assert middleware._create_summary(_long_history()) == "Summary."+ assert len(capped.prompts) == 2+ assert "turn-9" in capped.prompts[1]+ assert len(capped.prompts[1]) < len(uncapped.prompts[1])+++def test_summarizer_non_str_return_raises_type_error() -> None:+ middleware = SummarizationMiddleware(+ model=MockChatModel(),+ summarizer=lambda _: AIMessage(content="not a str"), # type: ignore[arg-type]+ )++ with pytest.raises(TypeError, match="`summarizer` must return a str, got AIMessage"):+ middleware._create_summary([HumanMessage(content="hi")])+++async def test_summarizer_non_str_return_raises_type_error_async() -> None:+ async def summarize(_: list[AnyMessage]) -> Any:+ return None++ middleware = SummarizationMiddleware(model=MockChatModel(), summarizer=summarize)++ with pytest.raises(TypeError, match="`summarizer` must return a str, got NoneType"):+ await middleware._acreate_summary([HumanMessage(content="hi")])+++async def test_sync_summarizer_on_async_path() -> None:+ def summarize(messages: list[AnyMessage]) -> str:+ return f"{len(messages)} messages"++ middleware = SummarizationMiddleware(model=_AlwaysFailingModel(), summarizer=summarize)++ assert await middleware._acreate_summary([HumanMessage(content="hi")]) == "1 messages"+++def test_async_summarizer_on_sync_path_raises() -> None:+ async def summarize(_: list[AnyMessage]) -> str:+ return "summary"++ middleware = SummarizationMiddleware(model=MockChatModel(), summarizer=summarize)++ with pytest.raises(TypeError, match="coroutine function synchronously"):+ middleware._create_summary([HumanMessage(content="hi")])