The agent engineering platform.
Python agentsaiai-agentsanthropicchatgptdeepagentsenterpriseframeworkgeminigenerative-ailangchainlanggraphllmmultiagentopen-sourceopenaipydanticpythonragtypescript
fix(langchain): preserve history on `SummarizationMiddleware` summary failure (#39268)
Nishitha Mbc16168d710d
diff --git a/libs/langchain_v1/langchain/agents/middleware/summarization.py b/libs/langchain_v1/langchain/agents/middleware/summarization.pyindex 0695693a84..407194087b 100644--- a/libs/langchain_v1/langchain/agents/middleware/summarization.py+++ b/libs/langchain_v1/langchain/agents/middleware/summarization.py@@ -1,5 +1,6 @@"""Summarization middleware."""+import loggingimport uuidimport warningsfrom collections.abc import Callable, Iterable, Mapping@@ -32,6 +33,8 @@ from langchain.agents.middleware.internal_call_transformer import (from langchain.agents.middleware.types import AgentMiddleware, AgentState, ContextT, ResponseTfrom langchain.chat_models import BaseChatModel, init_chat_model+logger = logging.getLogger(__name__)+TokenCounter = Callable[[Iterable[MessageLikeRepresentation]], int]DEFAULT_SUMMARY_PROMPT = """<role>@@ -232,6 +235,11 @@ class SummarizationMiddleware(AgentMiddleware[AgentState[ResponseT], ContextT, RThis middleware monitors message token counts and automatically summarizes oldermessages when a threshold is reached, preserving recent messages and maintainingcontext continuity by ensuring AI/Tool message pairs remain together.++ Transient summary-generation errors (rate limits, timeouts) are retried+ in-process, up to 3 attempts total, via `Runnable.with_retry`. If a summary call+ still fails after those attempts, the underlying error propagates rather than+ fabricating a summary."""transformers = (InternalCallTransformer,)@@ -346,6 +354,9 @@ class SummarizationMiddleware(AgentMiddleware[AgentState[ResponseT], ContextT, Rmodel = init_chat_model(model)self.model = model+ # Retry transient errors (rate limits, timeouts) in-process, up to 3 attempts+ # total, before giving up. See `Runnable.with_retry`.+ self._summary_model = self.model.with_retry()self.trigger: ContextSize | TriggerClause | list[ContextSize | TriggerClause] | None = (self._copy_trigger(trigger)@@ -395,6 +406,9 @@ class SummarizationMiddleware(AgentMiddleware[AgentState[ResponseT], ContextT, RReturns:An updated state with summarized messages if summarization was performed.++ Raises:+ Exception: If summary generation still fails once retries are exhausted."""messages = state["messages"]self._ensure_message_ids(messages)@@ -433,6 +447,9 @@ class SummarizationMiddleware(AgentMiddleware[AgentState[ResponseT], ContextT, RReturns:An updated state with summarized messages if summarization was performed.++ Raises:+ Exception: If summary generation still fails once retries are exhausted."""messages = state["messages"]self._ensure_message_ids(messages)@@ -817,6 +834,13 @@ class SummarizationMiddleware(AgentMiddleware[AgentState[ResponseT], ContextT, RArgs:messages_to_summarize: Messages to summarize.++ Returns:+ The generated summary.++ Raises:+ Exception: If the summary model call still fails once retries (see+ `Runnable.with_retry`) are exhausted."""if not messages_to_summarize:return "No previous conversation history."@@ -829,20 +853,24 @@ class SummarizationMiddleware(AgentMiddleware[AgentState[ResponseT], ContextT, R# prompt while excluding raw message metadata from the token budget.formatted_messages = get_buffer_string(trimmed_messages, format="xml")- try:- response = self.model.invoke(- self.summary_prompt.format(messages=formatted_messages).rstrip(),- config={"metadata": {"lc_source": "summarization", **internal_call_metadata()}},- )- return response.text.strip()- except Exception as e:- return f"Error generating summary: {e!s}"+ response = self._summary_model.invoke(+ self.summary_prompt.format(messages=formatted_messages).rstrip(),+ config={"metadata": {"lc_source": "summarization", **internal_call_metadata()}},+ )+ return response.text.strip()async def _acreate_summary(self, messages_to_summarize: list[AnyMessage]) -> str:"""Generate summary for the given messages.Args:messages_to_summarize: Messages to summarize.++ Returns:+ The generated summary.++ Raises:+ Exception: If the summary model call still fails once retries (see+ `Runnable.with_retry`) are exhausted."""if not messages_to_summarize:return "No previous conversation history."@@ -855,14 +883,11 @@ class SummarizationMiddleware(AgentMiddleware[AgentState[ResponseT], ContextT, R# prompt while excluding raw message metadata from the token budget.formatted_messages = get_buffer_string(trimmed_messages, format="xml")- try:- response = await self.model.ainvoke(- self.summary_prompt.format(messages=formatted_messages).rstrip(),- config={"metadata": {"lc_source": "summarization", **internal_call_metadata()}},- )- return response.text.strip()- except Exception as e:- return f"Error generating summary: {e!s}"+ response = await self._summary_model.ainvoke(+ self.summary_prompt.format(messages=formatted_messages).rstrip(),+ config={"metadata": {"lc_source": "summarization", **internal_call_metadata()}},+ )+ return response.text.strip()def _trim_messages_for_summary(self, messages: list[AnyMessage]) -> list[AnyMessage]:"""Trim messages to fit within summary generation limits."""diff --git a/libs/langchain_v1/tests/unit_tests/agents/middleware/implementations/test_summarization.py b/libs/langchain_v1/tests/unit_tests/agents/middleware/implementations/test_summarization.pyindex a01b94e327..1c6a34579f 100644--- a/libs/langchain_v1/tests/unit_tests/agents/middleware/implementations/test_summarization.py+++ b/libs/langchain_v1/tests/unit_tests/agents/middleware/implementations/test_summarization.py@@ -24,6 +24,7 @@ from langchain_core.messages import (from langchain_core.messages.utils import count_tokens_approximately, get_buffer_stringfrom langchain_core.outputs import ChatGeneration, ChatResultfrom langchain_core.runnables import RunnableConfig+from langgraph.checkpoint.memory import InMemorySaverfrom langgraph.graph.message import REMOVE_ALL_MESSAGESfrom langgraph.runtime import Runtimefrom pydantic import Field@@ -246,8 +247,11 @@ def test_summarization_middleware_summary_creation() -> None:return "mock"middleware_error = SummarizationMiddleware(model=ErrorModel(), trigger=("tokens", 1000))- summary = middleware_error._create_summary(messages)- assert "Error generating summary: Model error" in summary+ # Bypass the retry wrapper so this test isn't slowed by real backoff delay; retry+ # behavior itself is covered by test_summarization_middleware_retries_transient_failure_in_call.+ middleware_error._summary_model = middleware_error.model+ with pytest.raises(ValueError, match="Model error"):+ middleware_error._create_summary(messages)# Test we raise warning if max_tokens_before_summary or messages_to_keep is specifiedwith pytest.warns(DeprecationWarning, match="max_tokens_before_summary is deprecated"):@@ -256,6 +260,226 @@ def test_summarization_middleware_summary_creation() -> None:SummarizationMiddleware(model=MockChatModel(), messages_to_keep=5)+def test_summarization_middleware_retries_transient_failure_in_call() -> None:+ """A transient failure must be retried in-process via `Runnable.with_retry`.++ A single `_create_summary` call must succeed if the underlying model recovers+ within 3 attempts, without the caller ever observing a failure.+ """++ class FlakyModel(BaseChatModel):+ """Model that fails a fixed number of times, then succeeds."""++ fail_count: int = Field(default=0)+ attempts: int = Field(default=0)++ @override+ def _generate(+ self,+ messages: list[BaseMessage],+ stop: list[str] | None = None,+ run_manager: CallbackManagerForLLMRun | None = None,+ **kwargs: Any,+ ) -> ChatResult:+ self.attempts += 1+ if self.attempts <= self.fail_count:+ msg = "429 Too Many Requests (simulated transient failure)"+ raise RuntimeError(msg)+ return ChatResult(generations=[ChatGeneration(message=AIMessage(content="Summary."))])++ @property+ def _llm_type(self) -> str:+ return "mock"++ model = FlakyModel(fail_count=2)+ middleware = SummarizationMiddleware(model=model, trigger=("messages", 2))+ # No delay between attempts so the test runs fast and deterministically; the+ # retry count itself (3, `Runnable.with_retry`'s own default) is untouched.+ middleware._summary_model.wait_exponential_jitter = False # type: ignore[attr-defined]++ summary = middleware._create_summary([HumanMessage(content="hi")])++ assert summary == "Summary."+ assert model.attempts == 3+++class _AlwaysFailingModel(BaseChatModel):+ """Chat model whose sync and async summary calls always raise."""++ @override+ def _generate(+ self,+ messages: list[BaseMessage],+ stop: list[str] | None = None,+ run_manager: CallbackManagerForLLMRun | None = None,+ **kwargs: Any,+ ) -> ChatResult:+ msg = "429 Too Many Requests"+ raise RuntimeError(msg)++ @override+ async def _agenerate(+ self,+ messages: list[BaseMessage],+ stop: list[str] | None = None,+ run_manager: AsyncCallbackManagerForLLMRun | None = None,+ **kwargs: Any,+ ) -> ChatResult:+ msg = "429 Too Many Requests"+ raise RuntimeError(msg)++ @property+ def _llm_type(self) -> str:+ return "mock"+++def _skip_retry(middleware: SummarizationMiddleware) -> None:+ """Bypass the retry wrapper so failure-path tests aren't slowed by real backoff.++ Retry behavior itself is covered by+ `test_summarization_middleware_retries_transient_failure_in_call`.+ """+ middleware._summary_model = middleware.model+++def test_create_summary_raises_on_failure() -> None:+ """`_create_summary` must raise, never fabricate a fake summary string."""+ middleware = SummarizationMiddleware(model=_AlwaysFailingModel())+ _skip_retry(middleware)++ with pytest.raises(RuntimeError, match="429 Too Many Requests"):+ middleware._create_summary([HumanMessage(content="hi")])+++async def test_acreate_summary_raises_on_failure() -> None:+ """Async: `_acreate_summary` must raise on failure."""+ middleware = SummarizationMiddleware(model=_AlwaysFailingModel())+ _skip_retry(middleware)++ with pytest.raises(RuntimeError, match="429 Too Many Requests"):+ await middleware._acreate_summary([HumanMessage(content="hi")])+++def test_summarization_middleware_before_model_raises_on_summary_failure() -> None:+ """A failed summary generation must raise, never fabricate a summary or delete history."""+ middleware = SummarizationMiddleware(+ model=_AlwaysFailingModel(), trigger=("messages", 2), keep=("messages", 1)+ )+ _skip_retry(middleware)+ messages: list[AnyMessage] = [+ HumanMessage(content="hi", id="h0"),+ AIMessage(content="hello", id="a0"),+ HumanMessage(content="how are you", id="h1"),+ ]+ state = AgentState[Any](messages=messages)++ with pytest.raises(RuntimeError, match="429 Too Many Requests"):+ middleware.before_model(state, Runtime())++ # The original message list is untouched - no fabricated summary, no deletion.+ assert state["messages"] == messages+++async def test_summarization_middleware_abefore_model_raises_on_summary_failure() -> None:+ """Async: same raise-on-failure behavior as `before_model`."""+ middleware = SummarizationMiddleware(+ model=_AlwaysFailingModel(), trigger=("messages", 2), keep=("messages", 1)+ )+ _skip_retry(middleware)+ messages: list[AnyMessage] = [+ HumanMessage(content="hi", id="h0"),+ AIMessage(content="hello", id="a0"),+ HumanMessage(content="how are you", id="h1"),+ ]+ state = AgentState[Any](messages=messages)++ with pytest.raises(RuntimeError, match="429 Too Many Requests"):+ await middleware.abefore_model(state, Runtime())++ assert state["messages"] == messages+++def test_summarization_middleware_e2e_failure_raises_without_corrupting_history() -> None:+ """End-to-end regression test.++ Verifies that exhausted summary retries raise an error without corrupting+ checkpointed history, and that summarization succeeds on a later invocation+ once the model recovers.+ """++ class TemporarilyFailingSummaryModel(BaseChatModel):+ """Summary-only model that fails until `working` is set to `True`."""++ working: bool = Field(default=False)++ @override+ def _generate(+ self,+ messages: list[BaseMessage],+ stop: list[str] | None = None,+ run_manager: CallbackManagerForLLMRun | None = None,+ **kwargs: Any,+ ) -> ChatResult:+ if not self.working:+ msg = "429 Too Many Requests (simulated persistent failure)"+ raise RuntimeError(msg)+ summary = AIMessage(content="Summary of the conversation so far.")+ return ChatResult(generations=[ChatGeneration(message=summary)])++ @property+ def _llm_type(self) -> str:+ return "mock"++ main_model = FakeToolCallingModel()+ summary_model = TemporarilyFailingSummaryModel()+ middleware = SummarizationMiddleware(+ model=summary_model, trigger=("messages", 6), keep=("messages", 2)+ )+ _skip_retry(middleware)++ agent = create_agent(+ model=main_model,+ tools=[],+ middleware=[middleware],+ checkpointer=InMemorySaver(),+ )+ config: RunnableConfig = {"configurable": {"thread_id": "e2e-summary-failure"}}++ # Build up history below the ("messages", 6) trigger threshold.+ for i in range(3):+ agent.invoke({"messages": [HumanMessage(f"turn {i}")]}, config)++ messages_below_threshold = agent.get_state(config).values["messages"]+ assert len(messages_below_threshold) == 6++ # This turn triggers summarization, which fails and raises. The new human+ # message is checkpointed, but no AI response, summary, or message removal occurs.+ with pytest.raises(RuntimeError, match="429 Too Many Requests"):+ agent.invoke({"messages": [HumanMessage("turn 3")]}, config)++ messages_after_failed_summary = agent.get_state(config).values["messages"]+ assert messages_after_failed_summary[:-1] == messages_below_threshold+ assert messages_after_failed_summary[-1].content == "turn 3"+ assert not any(+ "Error generating summary" in (m.content or "") for m in messages_after_failed_summary+ )++ # Once the summarizer is fixed, retrying the same turn must succeed and+ # actually summarize, proving the earlier failure didn't corrupt anything.+ summary_model.working = True+ agent.invoke({"messages": [HumanMessage("turn 3")]}, config)++ messages_after_recovered_summary = agent.get_state(config).values["messages"]+ assert len(messages_after_recovered_summary) < len(messages_after_failed_summary)+ assert any(+ "summary of the conversation" in (m.content or "").lower()+ for m in messages_after_recovered_summary+ )+ assert not any(m.content == "turn 0" for m in messages_after_recovered_summary), (+ "history should actually be trimmed once summarization succeeds"+ )++def test_summarization_middleware_trim_limit_none_keeps_all_messages() -> None:"""Verify disabling trim limit preserves full message sequence."""messages: list[AnyMessage] = [HumanMessage(content=str(i)) for i in range(10)]@@ -941,9 +1165,10 @@ async def test_summarization_middleware_async_error_handling() -> None:return "mock"middleware = SummarizationMiddleware(model=ErrorAsyncModel(), trigger=("messages", 5))+ _skip_retry(middleware)messages: list[AnyMessage] = [HumanMessage(content="test")]- summary = await middleware._acreate_summary(messages)- assert "Error generating summary: Async model error" in summary+ with pytest.raises(ValueError, match="Async model error"):+ await middleware._acreate_summary(messages)def test_summarization_middleware_cutoff_at_boundary() -> None: