Skip to content

The agent engineering platform.

feat(langchain): accept `summarizer` in `SummarizationMiddleware` (#41225)

ccurmea0074fb7e5f3
diff --git a/libs/langchain_v1/langchain/agents/middleware/summarization.py b/libs/langchain_v1/langchain/agents/middleware/summarization.py
index 585de9ba95..a63857c139 100644
--- a/libs/langchain_v1/langchain/agents/middleware/summarization.py
+++ b/libs/langchain_v1/langchain/agents/middleware/summarization.py
@@ -3,7 +3,7 @@
import logging
import uuid
import warnings
-from collections.abc import Callable, Iterable, Mapping
+from collections.abc import Callable, Coroutine, Iterable, Mapping
from functools import partial
from typing import Any, Literal, TypedDict, cast
@@ -21,7 +21,8 @@ from langchain_core.messages.utils import (
get_buffer_string,
trim_messages,
)
-from langchain_core.runnables import RunnableConfig
+from langchain_core.runnables import Runnable, RunnableConfig
+from langchain_core.runnables.base import coerce_to_runnable
from langgraph.graph.message import (
REMOVE_ALL_MESSAGES,
)
@@ -40,6 +41,19 @@ logger = logging.getLogger(__name__)
TokenCounter = Callable[[Iterable[MessageLikeRepresentation]], int]
+Summarizer = (
+ str
+ | BaseChatModel
+ | Runnable[list[AnyMessage], str]
+ | Callable[[list[AnyMessage]], str]
+ | Callable[[list[AnyMessage]], Coroutine[Any, Any, str]]
+)
+"""Generates a summary from the messages selected for summarization.
+
+Either a chat model (or model string) prompted with the summary prompt, or a
+`Runnable` or callable taking the messages and returning the summary text.
+"""
+
DEFAULT_SUMMARY_PROMPT = """<role>
Context Extraction Assistant
</role>
@@ -121,6 +135,14 @@ def _summary_config() -> RunnableConfig:
return {"metadata": {"lc_source": "summarization", **internal_call_metadata()}}
+def _check_summary(summary: object) -> str:
+ """Validate and normalize the text returned by a custom `summarizer`."""
+ if not isinstance(summary, str):
+ msg = f"`summarizer` must return a str, got {type(summary).__name__}."
+ raise TypeError(msg)
+ return summary.strip()
+
+
def _provider_matches(message_provider: str, model_ls_provider: str | None) -> bool:
if model_ls_provider is None:
return False
@@ -250,6 +272,10 @@ class SummarizationMiddleware(AgentMiddleware[AgentState[ResponseT], ContextT, R
the model's context window (`ContextOverflowError`), the call is retried once with
the middle of the history cut out. If a summary call still fails, the
underlying error propagates rather than fabricating a summary.
+
+ To generate summaries with something other than a chat model, pass a
+ `summarizer`. The middleware still decides when to summarize and which
+ messages to summarize; only the summary text comes from the summarizer.
"""
transformers = (InternalCallTransformer,)
@@ -268,6 +294,7 @@ class SummarizationMiddleware(AgentMiddleware[AgentState[ResponseT], ContextT, R
token_counter: TokenCounter = count_tokens_approximately,
summary_prompt: str = DEFAULT_SUMMARY_PROMPT,
trim_tokens_to_summarize: int | None = _DEFAULT_TRIM_TOKEN_LIMIT,
+ summarizer: Summarizer | None = None,
**deprecated_kwargs: Any,
) -> None:
"""Initialize summarization middleware.
@@ -336,6 +363,47 @@ class SummarizationMiddleware(AgentMiddleware[AgentState[ResponseT], ContextT, R
the summarization call.
Pass `None` to skip trimming entirely.
+ summarizer: Optional replacement for the summarization model call.
+
+ `model` is still used for its profile (fractional limits) and
+ token counting, but is not called to write summaries.
+
+ - A chat model, or a model string resolved with
+ `init_chat_model`, writes summaries in place of `model`. It is
+ prompted with `summary_prompt` and retried like `model` would
+ be. Use this to summarize with a different model than the one
+ whose context window you're managing. To summarize with a chat
+ model, pass its name or a `BaseChatModel` instance; wrapped
+ models such as `model.with_config(...)` are treated as a
+ `Runnable` (below).
+ - A `Runnable` or function receives the messages to summarize
+ (after trimming) and returns the summary text. `summary_prompt`
+ is not used. Functions may be sync or async; an async function
+ requires async invocation (e.g. `ainvoke`), and a sync function
+ runs in a thread pool when invoked asynchronously.
+
+ !!! example
+
+ ```python
+ # Manage gpt-6-sol's context window, summarize with gpt-6-luna
+ SummarizationMiddleware(
+ model="openai:gpt-6-sol",
+ trigger=("fraction", 0.8),
+ summarizer="openai:gpt-6-luna",
+ )
+
+
+ # Summarize with a non-LLM service
+ async def summarize(messages: Sequence[BaseMessage]) -> str:
+ return await my_summarization_service(get_buffer_string(messages))
+
+
+ SummarizationMiddleware(
+ model="openai:gpt-6-luna",
+ trigger=("fraction", 0.8),
+ summarizer=summarize,
+ )
+ ```
"""
# Handle deprecated parameters
if "max_tokens_before_summary" in deprecated_kwargs:
@@ -389,6 +457,16 @@ class SummarizationMiddleware(AgentMiddleware[AgentState[ResponseT], ContextT, R
self._partial_token_counter = token_counter
self.summary_prompt = summary_prompt
self.trim_tokens_to_summarize = trim_tokens_to_summarize
+ if isinstance(summarizer, str):
+ summarizer = init_chat_model(summarizer)
+ # The chat model that writes summaries: `model` unless `summarizer` replaces it.
+ self._summary_chat_model = self.model
+ self.summarizer: Runnable[list[AnyMessage], str] | None = None
+ if isinstance(summarizer, BaseChatModel):
+ self._summary_chat_model = summarizer
+ self._summary_model = summarizer.with_retry(retry_if_exception_type=default_retry_on)
+ elif summarizer is not None:
+ self.summarizer = coerce_to_runnable(summarizer)
requires_profile = any("fraction" in clause for clause in self._trigger_clauses)
if self.keep[0] == "fraction":
@@ -726,10 +804,10 @@ class SummarizationMiddleware(AgentMiddleware[AgentState[ResponseT], ContextT, R
# Advance past any ToolMessages to avoid splitting AI/Tool pairs
return self._find_safe_cutoff_point(messages, cutoff_candidate)
- def _get_profile_limits(self) -> int | None:
- """Retrieve max input token limit from the model profile."""
+ def _get_profile_limits(self, model: BaseChatModel | None = None) -> int | None:
+ """Retrieve max input token limit from the profile of `model` (default: `self.model`)."""
try:
- profile = self.model.profile
+ profile = (model or self.model).profile
except AttributeError:
return None
@@ -858,6 +936,11 @@ class SummarizationMiddleware(AgentMiddleware[AgentState[ResponseT], ContextT, R
if not trimmed_messages:
return "Previous conversation was too long to summarize."
+ if self.summarizer is not None:
+ return _check_summary(
+ self.summarizer.invoke(trimmed_messages, config=_summary_config())
+ )
+
prompt = self._format_summary_prompt(trimmed_messages)
try:
response = self._summary_model.invoke(prompt, config=_summary_config())
@@ -888,6 +971,10 @@ class SummarizationMiddleware(AgentMiddleware[AgentState[ResponseT], ContextT, R
if not trimmed_messages:
return "Previous conversation was too long to summarize."
+ if self.summarizer is not None:
+ summary = await self.summarizer.ainvoke(trimmed_messages, config=_summary_config())
+ return _check_summary(summary)
+
prompt = self._format_summary_prompt(trimmed_messages)
try:
response = await self._summary_model.ainvoke(prompt, config=_summary_config())
@@ -910,7 +997,7 @@ class SummarizationMiddleware(AgentMiddleware[AgentState[ResponseT], ContextT, R
Cuts the middle out of the formatted history, keeping its start (where the
objective is usually stated) and its most recent part, within half of its
- token count and 40% of the model's `max_input_tokens` when known.
+ token count and 40% of the summary model's `max_input_tokens` when known.
Returns:
The retry prompt, or `None` if nothing fits the budget.
@@ -918,7 +1005,7 @@ class SummarizationMiddleware(AgentMiddleware[AgentState[ResponseT], ContextT, R
formatted = get_buffer_string(messages, format="xml")
tokens = self._partial_token_counter([HumanMessage(content=formatted)])
budget = tokens // 2
- if (max_input_tokens := self._get_profile_limits()) is not None:
+ if (max_input_tokens := self._get_profile_limits(self._summary_chat_model)) is not None:
# 40% rather than half absorbs up to 1.25x usage-metadata scaling.
budget = min(budget, max_input_tokens * 2 // 5)
keep_chars = len(formatted) * budget // max(tokens, 1)
diff --git a/libs/langchain_v1/tests/unit_tests/agents/middleware/implementations/test_summarization.py b/libs/langchain_v1/tests/unit_tests/agents/middleware/implementations/test_summarization.py
index 7315ba6a0d..acf00efffa 100644
--- a/libs/langchain_v1/tests/unit_tests/agents/middleware/implementations/test_summarization.py
+++ b/libs/langchain_v1/tests/unit_tests/agents/middleware/implementations/test_summarization.py
@@ -24,7 +24,7 @@ from langchain_core.messages import (
)
from langchain_core.messages.utils import count_tokens_approximately, get_buffer_string
from langchain_core.outputs import ChatGeneration, ChatResult
-from langchain_core.runnables import RunnableConfig
+from langchain_core.runnables import RunnableConfig, RunnableLambda
from langgraph.checkpoint.memory import InMemorySaver
from langgraph.graph.message import REMOVE_ALL_MESSAGES
from langgraph.runtime import Runtime
@@ -2473,3 +2473,269 @@ class TestSummarizationStreamingEndToEnd:
texts.append(text)
assert texts == ["Final answer."]
+
+
+def test_summarizer_replaces_model_call() -> None:
+ """A `summarizer` writes the summary; the model and `summary_prompt` are not used."""
+ received: list[list[AnyMessage]] = []
+
+ def summarizer(messages: list[AnyMessage]) -> str:
+ received.append(messages)
+ return " Custom summary "
+
+ middleware = SummarizationMiddleware(
+ model=_AlwaysFailingModel(),
+ trigger=("messages", 3),
+ keep=("messages", 1),
+ summary_prompt="unused {messages}",
+ summarizer=summarizer,
+ )
+ messages: list[AnyMessage] = [
+ HumanMessage(content="hi", id="h0"),
+ AIMessage(content="hello", id="a0"),
+ HumanMessage(content="how are you", id="h1"),
+ ]
+
+ result = middleware.before_model(AgentState[Any](messages=messages), Runtime())
+
+ assert result is not None
+ assert received == [messages[:2]]
+ assert isinstance(result["messages"][1], HumanMessage)
+ assert result["messages"][1].text.endswith("Custom summary")
+ assert result["messages"][2:] == messages[2:]
+
+
+def test_summarizer_receives_trimmed_messages() -> None:
+ """The summarizer sees the messages after `trim_tokens_to_summarize` is applied."""
+ received: list[list[AnyMessage]] = []
+
+ def summarizer(messages: list[AnyMessage]) -> str:
+ received.append(messages)
+ return "summary"
+
+ middleware = SummarizationMiddleware(
+ model=MockChatModel(), trim_tokens_to_summarize=None, summarizer=summarizer
+ )
+ messages: list[AnyMessage] = [HumanMessage(content=str(i)) for i in range(5)]
+ with patch.object(middleware, "_trim_messages_for_summary", return_value=messages[-2:]) as trim:
+ assert middleware._create_summary(messages) == "summary"
+
+ trim.assert_called_once_with(messages)
+ assert received == [messages[-2:]]
+
+
+def test_summarizer_accepts_runnable_and_gets_summarization_metadata() -> None:
+ """A `Runnable` summarizer is invoked with the summarization metadata."""
+ configs: list[RunnableConfig] = []
+
+ def summarize(messages: list[AnyMessage], config: RunnableConfig) -> str:
+ configs.append(config)
+ return get_buffer_string(messages)
+
+ middleware = SummarizationMiddleware(
+ model=_AlwaysFailingModel(), summarizer=RunnableLambda(summarize)
+ )
+
+ assert middleware._create_summary([HumanMessage(content="hi")]) == "Human: hi"
+ metadata = configs[0]["metadata"]
+ assert metadata["lc_source"] == "summarization"
+ assert (
+ metadata[INTERNAL_CALL_METADATA_KEY] == internal_call_metadata()[INTERNAL_CALL_METADATA_KEY]
+ )
+
+
+async def test_summarizer_async() -> None:
+ """Async invocation uses the summarizer's async implementation."""
+
+ async def summarize(messages: list[AnyMessage]) -> str:
+ return f"{len(messages)} messages"
+
+ middleware = SummarizationMiddleware(
+ model=_AlwaysFailingModel(),
+ trigger=("messages", 3),
+ keep=("messages", 1),
+ summarizer=summarize,
+ )
+ messages: list[AnyMessage] = [
+ HumanMessage(content="hi", id="h0"),
+ AIMessage(content="hello", id="a0"),
+ HumanMessage(content="how are you", id="h1"),
+ ]
+
+ result = await middleware.abefore_model(AgentState[Any](messages=messages), Runtime())
+
+ assert result is not None
+ assert result["messages"][1].text.endswith("2 messages")
+
+
+def test_summarizer_errors_propagate() -> None:
+ """Errors raised by the summarizer propagate and leave history untouched."""
+
+ def summarizer(_: list[AnyMessage]) -> str:
+ msg = "summarizer down"
+ raise RuntimeError(msg)
+
+ middleware = SummarizationMiddleware(
+ model=MockChatModel(),
+ trigger=("messages", 2),
+ keep=("messages", 1),
+ summarizer=summarizer,
+ )
+ messages: list[AnyMessage] = [
+ HumanMessage(content="hi", id="h0"),
+ AIMessage(content="hello", id="a0"),
+ ]
+ state = AgentState[Any](messages=messages)
+
+ with pytest.raises(RuntimeError, match="summarizer down"):
+ middleware.before_model(state, Runtime())
+ assert state["messages"] == messages
+
+
+def test_summarizer_empty_history_skips_summarizer() -> None:
+ """The built-in empty-history message is returned without calling the summarizer."""
+
+ def summarizer(_: list[AnyMessage]) -> str:
+ msg = "should not be called"
+ raise AssertionError(msg)
+
+ middleware = SummarizationMiddleware(model=MockChatModel(), summarizer=summarizer)
+
+ assert middleware._create_summary([]) == "No previous conversation history."
+
+
+def test_chat_model_summarizer_uses_summary_prompt() -> None:
+ """A chat model `summarizer` writes summaries with `summary_prompt`; `model` isn't called."""
+ summary_model = _OverflowingModel(max_chars=1_000_000)
+ middleware = SummarizationMiddleware(
+ model=_AlwaysFailingModel(),
+ summary_prompt="Summarize this: {messages}",
+ summarizer=summary_model,
+ )
+
+ assert middleware._create_summary([HumanMessage(content="hi")]) == "Summary."
+ assert len(summary_model.prompts) == 1
+ assert summary_model.prompts[0].startswith("Summarize this: ")
+ assert "hi" in summary_model.prompts[0]
+ assert middleware.summarizer is None
+
+
+async def test_chat_model_summarizer_async() -> None:
+ summary_model = _OverflowingModel(max_chars=1_000_000)
+ middleware = SummarizationMiddleware(model=_AlwaysFailingModel(), summarizer=summary_model)
+
+ assert await middleware._acreate_summary([HumanMessage(content="hi")]) == "Summary."
+ assert len(summary_model.prompts) == 1
+
+
+def test_chat_model_summarizer_keeps_profile_of_model() -> None:
+ """Fractional limits still come from `model`'s profile, not the summarizer's."""
+ summary_model = _OverflowingModel(max_chars=1_000_000)
+ middleware = SummarizationMiddleware(
+ model=ProfileChatModel(),
+ trigger=("fraction", 0.5),
+ summarizer=summary_model,
+ )
+
+ assert middleware._get_profile_limits() == 1000
+ assert middleware.model is not summary_model
+
+
+def test_string_summarizer_is_resolved_with_init_chat_model() -> None:
+ summary_model = _OverflowingModel(max_chars=1_000_000)
+ with patch(
+ "langchain.agents.middleware.summarization.init_chat_model",
+ return_value=summary_model,
+ ) as init:
+ middleware = SummarizationMiddleware(
+ model=_AlwaysFailingModel(), summarizer="openai:gpt-5-mini"
+ )
+
+ init.assert_called_once_with("openai:gpt-5-mini")
+ assert middleware._create_summary([HumanMessage(content="hi")]) == "Summary."
+
+
+def test_chat_model_summarizer_does_not_retry_non_retryable_errors() -> None:
+ class AuthFailingModel(BaseChatModel):
+ attempts: int = 0
+
+ @override
+ def _generate(
+ self,
+ messages: list[BaseMessage],
+ stop: list[str] | None = None,
+ run_manager: CallbackManagerForLLMRun | None = None,
+ **kwargs: Any,
+ ) -> ChatResult:
+ self.attempts += 1
+ msg = "invalid api key"
+ raise ModelAuthenticationError(msg)
+
+ @property
+ def _llm_type(self) -> str:
+ return "mock"
+
+ summary_model = AuthFailingModel()
+ middleware = SummarizationMiddleware(model=MockChatModel(), summarizer=summary_model)
+ middleware._summary_model.wait_exponential_jitter = False # type: ignore[attr-defined]
+
+ with pytest.raises(ModelAuthenticationError):
+ middleware._create_summary([HumanMessage(content="hi")])
+ assert summary_model.attempts == 1
+
+
+def test_chat_model_summarizer_overflow_budget_uses_summarizer_profile() -> None:
+ """The overflow retry is sized to the summary model's context window."""
+ uncapped = _OverflowingModel(max_chars=25_000)
+ SummarizationMiddleware(
+ model=MockChatModel(), trim_tokens_to_summarize=None, summarizer=uncapped
+ )._create_summary(_long_history())
+
+ capped = _OverflowingModel(max_chars=25_000, profile=ModelProfile(max_input_tokens=1000))
+ middleware = SummarizationMiddleware(
+ model=MockChatModel(), trim_tokens_to_summarize=None, summarizer=capped
+ )
+
+ assert middleware._create_summary(_long_history()) == "Summary."
+ assert len(capped.prompts) == 2
+ assert "turn-9" in capped.prompts[1]
+ assert len(capped.prompts[1]) < len(uncapped.prompts[1])
+
+
+def test_summarizer_non_str_return_raises_type_error() -> None:
+ middleware = SummarizationMiddleware(
+ model=MockChatModel(),
+ summarizer=lambda _: AIMessage(content="not a str"), # type: ignore[arg-type]
+ )
+
+ with pytest.raises(TypeError, match="`summarizer` must return a str, got AIMessage"):
+ middleware._create_summary([HumanMessage(content="hi")])
+
+
+async def test_summarizer_non_str_return_raises_type_error_async() -> None:
+ async def summarize(_: list[AnyMessage]) -> Any:
+ return None
+
+ middleware = SummarizationMiddleware(model=MockChatModel(), summarizer=summarize)
+
+ with pytest.raises(TypeError, match="`summarizer` must return a str, got NoneType"):
+ await middleware._acreate_summary([HumanMessage(content="hi")])
+
+
+async def test_sync_summarizer_on_async_path() -> None:
+ def summarize(messages: list[AnyMessage]) -> str:
+ return f"{len(messages)} messages"
+
+ middleware = SummarizationMiddleware(model=_AlwaysFailingModel(), summarizer=summarize)
+
+ assert await middleware._acreate_summary([HumanMessage(content="hi")]) == "1 messages"
+
+
+def test_async_summarizer_on_sync_path_raises() -> None:
+ async def summarize(_: list[AnyMessage]) -> str:
+ return "summary"
+
+ middleware = SummarizationMiddleware(model=MockChatModel(), summarizer=summarize)
+
+ with pytest.raises(TypeError, match="coroutine function synchronously"):
+ middleware._create_summary([HumanMessage(content="hi")])