Skip to content

The agent engineering platform.

fix(langchain): preserve history on `SummarizationMiddleware` summary failure (#39268)

Nishitha Mbc16168d710d
diff --git a/libs/langchain_v1/langchain/agents/middleware/summarization.py b/libs/langchain_v1/langchain/agents/middleware/summarization.py
index 0695693a84..407194087b 100644
--- a/libs/langchain_v1/langchain/agents/middleware/summarization.py
+++ b/libs/langchain_v1/langchain/agents/middleware/summarization.py
@@ -1,5 +1,6 @@
"""Summarization middleware."""
+import logging
import uuid
import warnings
from collections.abc import Callable, Iterable, Mapping
@@ -32,6 +33,8 @@ from langchain.agents.middleware.internal_call_transformer import (
from langchain.agents.middleware.types import AgentMiddleware, AgentState, ContextT, ResponseT
from langchain.chat_models import BaseChatModel, init_chat_model
+logger = logging.getLogger(__name__)
+
TokenCounter = Callable[[Iterable[MessageLikeRepresentation]], int]
DEFAULT_SUMMARY_PROMPT = """<role>
@@ -232,6 +235,11 @@ class SummarizationMiddleware(AgentMiddleware[AgentState[ResponseT], ContextT, R
This middleware monitors message token counts and automatically summarizes older
messages when a threshold is reached, preserving recent messages and maintaining
context continuity by ensuring AI/Tool message pairs remain together.
+
+ Transient summary-generation errors (rate limits, timeouts) are retried
+ in-process, up to 3 attempts total, via `Runnable.with_retry`. If a summary call
+ still fails after those attempts, the underlying error propagates rather than
+ fabricating a summary.
"""
transformers = (InternalCallTransformer,)
@@ -346,6 +354,9 @@ class SummarizationMiddleware(AgentMiddleware[AgentState[ResponseT], ContextT, R
model = init_chat_model(model)
self.model = model
+ # Retry transient errors (rate limits, timeouts) in-process, up to 3 attempts
+ # total, before giving up. See `Runnable.with_retry`.
+ self._summary_model = self.model.with_retry()
self.trigger: ContextSize | TriggerClause | list[ContextSize | TriggerClause] | None = (
self._copy_trigger(trigger)
@@ -395,6 +406,9 @@ class SummarizationMiddleware(AgentMiddleware[AgentState[ResponseT], ContextT, R
Returns:
An updated state with summarized messages if summarization was performed.
+
+ Raises:
+ Exception: If summary generation still fails once retries are exhausted.
"""
messages = state["messages"]
self._ensure_message_ids(messages)
@@ -433,6 +447,9 @@ class SummarizationMiddleware(AgentMiddleware[AgentState[ResponseT], ContextT, R
Returns:
An updated state with summarized messages if summarization was performed.
+
+ Raises:
+ Exception: If summary generation still fails once retries are exhausted.
"""
messages = state["messages"]
self._ensure_message_ids(messages)
@@ -817,6 +834,13 @@ class SummarizationMiddleware(AgentMiddleware[AgentState[ResponseT], ContextT, R
Args:
messages_to_summarize: Messages to summarize.
+
+ Returns:
+ The generated summary.
+
+ Raises:
+ Exception: If the summary model call still fails once retries (see
+ `Runnable.with_retry`) are exhausted.
"""
if not messages_to_summarize:
return "No previous conversation history."
@@ -829,20 +853,24 @@ class SummarizationMiddleware(AgentMiddleware[AgentState[ResponseT], ContextT, R
# prompt while excluding raw message metadata from the token budget.
formatted_messages = get_buffer_string(trimmed_messages, format="xml")
- try:
- response = self.model.invoke(
- self.summary_prompt.format(messages=formatted_messages).rstrip(),
- config={"metadata": {"lc_source": "summarization", **internal_call_metadata()}},
- )
- return response.text.strip()
- except Exception as e:
- return f"Error generating summary: {e!s}"
+ response = self._summary_model.invoke(
+ self.summary_prompt.format(messages=formatted_messages).rstrip(),
+ config={"metadata": {"lc_source": "summarization", **internal_call_metadata()}},
+ )
+ return response.text.strip()
async def _acreate_summary(self, messages_to_summarize: list[AnyMessage]) -> str:
"""Generate summary for the given messages.
Args:
messages_to_summarize: Messages to summarize.
+
+ Returns:
+ The generated summary.
+
+ Raises:
+ Exception: If the summary model call still fails once retries (see
+ `Runnable.with_retry`) are exhausted.
"""
if not messages_to_summarize:
return "No previous conversation history."
@@ -855,14 +883,11 @@ class SummarizationMiddleware(AgentMiddleware[AgentState[ResponseT], ContextT, R
# prompt while excluding raw message metadata from the token budget.
formatted_messages = get_buffer_string(trimmed_messages, format="xml")
- try:
- response = await self.model.ainvoke(
- self.summary_prompt.format(messages=formatted_messages).rstrip(),
- config={"metadata": {"lc_source": "summarization", **internal_call_metadata()}},
- )
- return response.text.strip()
- except Exception as e:
- return f"Error generating summary: {e!s}"
+ response = await self._summary_model.ainvoke(
+ self.summary_prompt.format(messages=formatted_messages).rstrip(),
+ config={"metadata": {"lc_source": "summarization", **internal_call_metadata()}},
+ )
+ return response.text.strip()
def _trim_messages_for_summary(self, messages: list[AnyMessage]) -> list[AnyMessage]:
"""Trim messages to fit within summary generation limits."""
diff --git a/libs/langchain_v1/tests/unit_tests/agents/middleware/implementations/test_summarization.py b/libs/langchain_v1/tests/unit_tests/agents/middleware/implementations/test_summarization.py
index a01b94e327..1c6a34579f 100644
--- a/libs/langchain_v1/tests/unit_tests/agents/middleware/implementations/test_summarization.py
+++ b/libs/langchain_v1/tests/unit_tests/agents/middleware/implementations/test_summarization.py
@@ -24,6 +24,7 @@ from langchain_core.messages import (
from langchain_core.messages.utils import count_tokens_approximately, get_buffer_string
from langchain_core.outputs import ChatGeneration, ChatResult
from langchain_core.runnables import RunnableConfig
+from langgraph.checkpoint.memory import InMemorySaver
from langgraph.graph.message import REMOVE_ALL_MESSAGES
from langgraph.runtime import Runtime
from pydantic import Field
@@ -246,8 +247,11 @@ def test_summarization_middleware_summary_creation() -> None:
return "mock"
middleware_error = SummarizationMiddleware(model=ErrorModel(), trigger=("tokens", 1000))
- summary = middleware_error._create_summary(messages)
- assert "Error generating summary: Model error" in summary
+ # Bypass the retry wrapper so this test isn't slowed by real backoff delay; retry
+ # behavior itself is covered by test_summarization_middleware_retries_transient_failure_in_call.
+ middleware_error._summary_model = middleware_error.model
+ with pytest.raises(ValueError, match="Model error"):
+ middleware_error._create_summary(messages)
# Test we raise warning if max_tokens_before_summary or messages_to_keep is specified
with pytest.warns(DeprecationWarning, match="max_tokens_before_summary is deprecated"):
@@ -256,6 +260,226 @@ def test_summarization_middleware_summary_creation() -> None:
SummarizationMiddleware(model=MockChatModel(), messages_to_keep=5)
+def test_summarization_middleware_retries_transient_failure_in_call() -> None:
+ """A transient failure must be retried in-process via `Runnable.with_retry`.
+
+ A single `_create_summary` call must succeed if the underlying model recovers
+ within 3 attempts, without the caller ever observing a failure.
+ """
+
+ class FlakyModel(BaseChatModel):
+ """Model that fails a fixed number of times, then succeeds."""
+
+ fail_count: int = Field(default=0)
+ attempts: int = Field(default=0)
+
+ @override
+ def _generate(
+ self,
+ messages: list[BaseMessage],
+ stop: list[str] | None = None,
+ run_manager: CallbackManagerForLLMRun | None = None,
+ **kwargs: Any,
+ ) -> ChatResult:
+ self.attempts += 1
+ if self.attempts <= self.fail_count:
+ msg = "429 Too Many Requests (simulated transient failure)"
+ raise RuntimeError(msg)
+ return ChatResult(generations=[ChatGeneration(message=AIMessage(content="Summary."))])
+
+ @property
+ def _llm_type(self) -> str:
+ return "mock"
+
+ model = FlakyModel(fail_count=2)
+ middleware = SummarizationMiddleware(model=model, trigger=("messages", 2))
+ # No delay between attempts so the test runs fast and deterministically; the
+ # retry count itself (3, `Runnable.with_retry`'s own default) is untouched.
+ middleware._summary_model.wait_exponential_jitter = False # type: ignore[attr-defined]
+
+ summary = middleware._create_summary([HumanMessage(content="hi")])
+
+ assert summary == "Summary."
+ assert model.attempts == 3
+
+
+class _AlwaysFailingModel(BaseChatModel):
+ """Chat model whose sync and async summary calls always raise."""
+
+ @override
+ def _generate(
+ self,
+ messages: list[BaseMessage],
+ stop: list[str] | None = None,
+ run_manager: CallbackManagerForLLMRun | None = None,
+ **kwargs: Any,
+ ) -> ChatResult:
+ msg = "429 Too Many Requests"
+ raise RuntimeError(msg)
+
+ @override
+ async def _agenerate(
+ self,
+ messages: list[BaseMessage],
+ stop: list[str] | None = None,
+ run_manager: AsyncCallbackManagerForLLMRun | None = None,
+ **kwargs: Any,
+ ) -> ChatResult:
+ msg = "429 Too Many Requests"
+ raise RuntimeError(msg)
+
+ @property
+ def _llm_type(self) -> str:
+ return "mock"
+
+
+def _skip_retry(middleware: SummarizationMiddleware) -> None:
+ """Bypass the retry wrapper so failure-path tests aren't slowed by real backoff.
+
+ Retry behavior itself is covered by
+ `test_summarization_middleware_retries_transient_failure_in_call`.
+ """
+ middleware._summary_model = middleware.model
+
+
+def test_create_summary_raises_on_failure() -> None:
+ """`_create_summary` must raise, never fabricate a fake summary string."""
+ middleware = SummarizationMiddleware(model=_AlwaysFailingModel())
+ _skip_retry(middleware)
+
+ with pytest.raises(RuntimeError, match="429 Too Many Requests"):
+ middleware._create_summary([HumanMessage(content="hi")])
+
+
+async def test_acreate_summary_raises_on_failure() -> None:
+ """Async: `_acreate_summary` must raise on failure."""
+ middleware = SummarizationMiddleware(model=_AlwaysFailingModel())
+ _skip_retry(middleware)
+
+ with pytest.raises(RuntimeError, match="429 Too Many Requests"):
+ await middleware._acreate_summary([HumanMessage(content="hi")])
+
+
+def test_summarization_middleware_before_model_raises_on_summary_failure() -> None:
+ """A failed summary generation must raise, never fabricate a summary or delete history."""
+ middleware = SummarizationMiddleware(
+ model=_AlwaysFailingModel(), trigger=("messages", 2), keep=("messages", 1)
+ )
+ _skip_retry(middleware)
+ messages: list[AnyMessage] = [
+ HumanMessage(content="hi", id="h0"),
+ AIMessage(content="hello", id="a0"),
+ HumanMessage(content="how are you", id="h1"),
+ ]
+ state = AgentState[Any](messages=messages)
+
+ with pytest.raises(RuntimeError, match="429 Too Many Requests"):
+ middleware.before_model(state, Runtime())
+
+ # The original message list is untouched - no fabricated summary, no deletion.
+ assert state["messages"] == messages
+
+
+async def test_summarization_middleware_abefore_model_raises_on_summary_failure() -> None:
+ """Async: same raise-on-failure behavior as `before_model`."""
+ middleware = SummarizationMiddleware(
+ model=_AlwaysFailingModel(), trigger=("messages", 2), keep=("messages", 1)
+ )
+ _skip_retry(middleware)
+ messages: list[AnyMessage] = [
+ HumanMessage(content="hi", id="h0"),
+ AIMessage(content="hello", id="a0"),
+ HumanMessage(content="how are you", id="h1"),
+ ]
+ state = AgentState[Any](messages=messages)
+
+ with pytest.raises(RuntimeError, match="429 Too Many Requests"):
+ await middleware.abefore_model(state, Runtime())
+
+ assert state["messages"] == messages
+
+
+def test_summarization_middleware_e2e_failure_raises_without_corrupting_history() -> None:
+ """End-to-end regression test.
+
+ Verifies that exhausted summary retries raise an error without corrupting
+ checkpointed history, and that summarization succeeds on a later invocation
+ once the model recovers.
+ """
+
+ class TemporarilyFailingSummaryModel(BaseChatModel):
+ """Summary-only model that fails until `working` is set to `True`."""
+
+ working: bool = Field(default=False)
+
+ @override
+ def _generate(
+ self,
+ messages: list[BaseMessage],
+ stop: list[str] | None = None,
+ run_manager: CallbackManagerForLLMRun | None = None,
+ **kwargs: Any,
+ ) -> ChatResult:
+ if not self.working:
+ msg = "429 Too Many Requests (simulated persistent failure)"
+ raise RuntimeError(msg)
+ summary = AIMessage(content="Summary of the conversation so far.")
+ return ChatResult(generations=[ChatGeneration(message=summary)])
+
+ @property
+ def _llm_type(self) -> str:
+ return "mock"
+
+ main_model = FakeToolCallingModel()
+ summary_model = TemporarilyFailingSummaryModel()
+ middleware = SummarizationMiddleware(
+ model=summary_model, trigger=("messages", 6), keep=("messages", 2)
+ )
+ _skip_retry(middleware)
+
+ agent = create_agent(
+ model=main_model,
+ tools=[],
+ middleware=[middleware],
+ checkpointer=InMemorySaver(),
+ )
+ config: RunnableConfig = {"configurable": {"thread_id": "e2e-summary-failure"}}
+
+ # Build up history below the ("messages", 6) trigger threshold.
+ for i in range(3):
+ agent.invoke({"messages": [HumanMessage(f"turn {i}")]}, config)
+
+ messages_below_threshold = agent.get_state(config).values["messages"]
+ assert len(messages_below_threshold) == 6
+
+ # This turn triggers summarization, which fails and raises. The new human
+ # message is checkpointed, but no AI response, summary, or message removal occurs.
+ with pytest.raises(RuntimeError, match="429 Too Many Requests"):
+ agent.invoke({"messages": [HumanMessage("turn 3")]}, config)
+
+ messages_after_failed_summary = agent.get_state(config).values["messages"]
+ assert messages_after_failed_summary[:-1] == messages_below_threshold
+ assert messages_after_failed_summary[-1].content == "turn 3"
+ assert not any(
+ "Error generating summary" in (m.content or "") for m in messages_after_failed_summary
+ )
+
+ # Once the summarizer is fixed, retrying the same turn must succeed and
+ # actually summarize, proving the earlier failure didn't corrupt anything.
+ summary_model.working = True
+ agent.invoke({"messages": [HumanMessage("turn 3")]}, config)
+
+ messages_after_recovered_summary = agent.get_state(config).values["messages"]
+ assert len(messages_after_recovered_summary) < len(messages_after_failed_summary)
+ assert any(
+ "summary of the conversation" in (m.content or "").lower()
+ for m in messages_after_recovered_summary
+ )
+ assert not any(m.content == "turn 0" for m in messages_after_recovered_summary), (
+ "history should actually be trimmed once summarization succeeds"
+ )
+
+
def test_summarization_middleware_trim_limit_none_keeps_all_messages() -> None:
"""Verify disabling trim limit preserves full message sequence."""
messages: list[AnyMessage] = [HumanMessage(content=str(i)) for i in range(10)]
@@ -941,9 +1165,10 @@ async def test_summarization_middleware_async_error_handling() -> None:
return "mock"
middleware = SummarizationMiddleware(model=ErrorAsyncModel(), trigger=("messages", 5))
+ _skip_retry(middleware)
messages: list[AnyMessage] = [HumanMessage(content="test")]
- summary = await middleware._acreate_summary(messages)
- assert "Error generating summary: Async model error" in summary
+ with pytest.raises(ValueError, match="Async model error"):
+ await middleware._acreate_summary(messages)
def test_summarization_middleware_cutoff_at_boundary() -> None: