Platform for stateful agents: AI with advanced memory that can learn and self-improve over time.
feat: refactor summarization and message persistence code [LET-6464] (#6561)
Sarah Woodersbbd52e291c30
diff --git a/fern/openapi.json b/fern/openapi.jsonindex fb51e8c3b..b7e185467 100644--- a/fern/openapi.json+++ b/fern/openapi.json@@ -35585,7 +35585,8 @@"no_tool_call","tool_rule","cancelled",- "requires_approval"+ "requires_approval",+ "context_window_overflow_in_system_prompt"],"title": "StopReasonType"},diff --git a/letta/agents/helpers.py b/letta/agents/helpers.pyindex 1c6b2570d..f0ec2c4f9 100644--- a/letta/agents/helpers.py+++ b/letta/agents/helpers.py@@ -172,7 +172,6 @@ async def _prepare_in_context_messages_no_persist_async(new_in_context_messages.extend(follow_up_messages)else:# User is trying to send a regular message- # if current_in_context_messages and current_in_context_messages[-1].role == "approval":if current_in_context_messages and current_in_context_messages[-1].is_approval_request():raise PendingApprovalError(pending_request_id=current_in_context_messages[-1].id)diff --git a/letta/agents/letta_agent_v3.py b/letta/agents/letta_agent_v3.pyindex 9b6175bb0..d44463c85 100644--- a/letta/agents/letta_agent_v3.py+++ b/letta/agents/letta_agent_v3.py@@ -20,7 +20,7 @@ from letta.agents.helpers import ()from letta.agents.letta_agent_v2 import LettaAgentV2from letta.constants import DEFAULT_MAX_STEPS, NON_USER_MSG_PREFIX, REQUEST_HEARTBEAT_PARAM, SUMMARIZATION_TRIGGER_MULTIPLIER-from letta.errors import ContextWindowExceededError, LLMError+from letta.errors import ContextWindowExceededError, LLMError, SystemPromptTokenExceededErrorfrom letta.helpers import ToolRulesSolverfrom letta.helpers.datetime_helpers import get_utc_time, get_utc_timestamp_nsfrom letta.helpers.message_helper import convert_message_creates_to_messages@@ -78,6 +78,7 @@ class LettaAgentV3(LettaAgentV2):# from per-step usage but can be updated after summarization without# affecting step-level telemetry.self.context_token_estimate: int | None = None+ self.in_context_messages: list[Message] = [] # in-memory trackerdef _compute_tool_return_truncation_chars(self) -> int:"""Compute a dynamic cap for tool returns in requests.@@ -119,7 +120,7 @@ class LettaAgentV3(LettaAgentV2):request_span = self._request_checkpoint_start(request_start_timestamp_ns=request_start_timestamp_ns)response_letta_messages = []- in_context_messages, input_messages_to_persist = await _prepare_in_context_messages_no_persist_async(+ curr_in_context_messages, input_messages_to_persist = await _prepare_in_context_messages_no_persist_async(input_messages, self.agent_state, self.message_manager, self.actor, run_id)follow_up_messages = []@@ -127,13 +128,15 @@ class LettaAgentV3(LettaAgentV2):follow_up_messages = input_messages_to_persist[1:]input_messages_to_persist = [input_messages_to_persist[0]]- in_context_messages = in_context_messages + input_messages_to_persist+ self.in_context_messages = curr_in_context_messagesfor i in range(max_steps):if i == 1 and follow_up_messages:input_messages_to_persist = follow_up_messagesfollow_up_messages = []+response = self._step(- messages=in_context_messages + self.response_messages,+ # we append input_messages_to_persist since they aren't checkpointed as in-context until the end of the step (may be rolled back)+ messages=list(self.in_context_messages + input_messages_to_persist),input_messages_to_persist=input_messages_to_persist,# TODO need to support non-streaming adapter toollm_adapter=SimpleLLMRequestAdapter(llm_client=self.llm_client, llm_config=self.agent_state.llm_config),@@ -142,6 +145,7 @@ class LettaAgentV3(LettaAgentV2):include_return_message_types=include_return_message_types,request_start_timestamp_ns=request_start_timestamp_ns,)+ input_messages_to_persist = [] # clear after first stepasync for chunk in response:response_letta_messages.append(chunk)@@ -150,53 +154,65 @@ class LettaAgentV3(LettaAgentV2):if not self.should_continue and self.stop_reason.stop_reason == StopReasonType.cancelled.value:break- # Proactive summarization if approaching context limit- if (- self.context_token_estimate is not None- and self.context_token_estimate > self.agent_state.llm_config.context_window * SUMMARIZATION_TRIGGER_MULTIPLIER- and not self.agent_state.message_buffer_autoclear- ):- self.logger.warning(- f"Step usage ({self.last_step_usage.total_tokens} tokens) approaching "- f"context limit ({self.agent_state.llm_config.context_window}), triggering summarization."- )-- in_context_messages = await self.summarize_conversation_history(- in_context_messages=in_context_messages,- new_letta_messages=self.response_messages,- total_tokens=self.context_token_estimate,- force=True,- )-- # Clear to avoid duplication in next iteration- self.response_messages = []+ # TODO: persist the input messages if successful first step completion+ # TODO: persist the new messages / step / run++ ## Proactive summarization if approaching context limit+ # if (+ # self.context_token_estimate is not None+ # and self.context_token_estimate > self.agent_state.llm_config.context_window * SUMMARIZATION_TRIGGER_MULTIPLIER+ # and not self.agent_state.message_buffer_autoclear+ # ):+ # self.logger.warning(+ # f"Step usage ({self.last_step_usage.total_tokens} tokens) approaching "+ # f"context limit ({self.agent_state.llm_config.context_window}), triggering summarization."+ # )++ # in_context_messages = await self.summarize_conversation_history(+ # in_context_messages=in_context_messages,+ # new_letta_messages=self.response_messages,+ # total_tokens=self.context_token_estimate,+ # force=True,+ # )++ # # Clear to avoid duplication in next iteration+ # self.response_messages = []if not self.should_continue:break- input_messages_to_persist = []+ # input_messages_to_persist = []if i == max_steps - 1 and self.stop_reason is None:self.stop_reason = LettaStopReason(stop_reason=StopReasonType.max_steps.value)- # Rebuild context window after stepping (safety net)- if not self.agent_state.message_buffer_autoclear:- if self.context_token_estimate is not None:- await self.summarize_conversation_history(- in_context_messages=in_context_messages,- new_letta_messages=self.response_messages,- total_tokens=self.context_token_estimate,- force=False,- )- else:- self.logger.warning(- "Post-loop summarization skipped: last_step_usage is None. "- "No step completed successfully or usage stats were not updated."- )+ ## Rebuild context window after stepping (safety net)+ # if not self.agent_state.message_buffer_autoclear:+ # if self.context_token_estimate is not None:+ # await self.summarize_conversation_history(+ # in_context_messages=in_context_messages,+ # new_letta_messages=self.response_messages,+ # total_tokens=self.context_token_estimate,+ # force=False,+ # )+ # else:+ # self.logger.warning(+ # "Post-loop summarization skipped: last_step_usage is None. "+ # "No step completed successfully or usage stats were not updated."+ # )if self.stop_reason is None:self.stop_reason = LettaStopReason(stop_reason=StopReasonType.end_turn.value)+ # construct the response+ response_letta_messages = Message.to_letta_messages_from_list(+ self.response_messages,+ use_assistant_message=False, # NOTE: set to false+ reverse=False,+ text_is_assistant_message=True,+ )+ if include_return_message_types:+ response_letta_messages = [m for m in response_letta_messages if m.message_type in include_return_message_types]result = LettaResponse(messages=response_letta_messages, stop_reason=self.stop_reason, usage=self.usage)if run_id:if self.job_update_metadata is None:@@ -265,13 +281,14 @@ class LettaAgentV3(LettaAgentV2):follow_up_messages = input_messages_to_persist[1:]input_messages_to_persist = [input_messages_to_persist[0]]- in_context_messages = in_context_messages + input_messages_to_persist+ self.in_context_messages = in_context_messagesfor i in range(max_steps):if i == 1 and follow_up_messages:input_messages_to_persist = follow_up_messagesfollow_up_messages = []response = self._step(- messages=in_context_messages + self.response_messages,+ # we append input_messages_to_persist since they aren't checkpointed as in-context until the end of the step (may be rolled back)+ messages=list(self.in_context_messages + input_messages_to_persist),input_messages_to_persist=input_messages_to_persist,llm_adapter=llm_adapter,run_id=run_id,@@ -279,6 +296,7 @@ class LettaAgentV3(LettaAgentV2):include_return_message_types=include_return_message_types,request_start_timestamp_ns=request_start_timestamp_ns,)+ input_messages_to_persist = [] # clear after first stepasync for chunk in response:response_letta_messages.append(chunk)if first_chunk:@@ -290,49 +308,29 @@ class LettaAgentV3(LettaAgentV2):if not self.should_continue and self.stop_reason.stop_reason == StopReasonType.cancelled.value:break- # Proactive summarization if approaching context limit- if (- self.context_token_estimate is not None- and self.context_token_estimate > self.agent_state.llm_config.context_window * SUMMARIZATION_TRIGGER_MULTIPLIER- and not self.agent_state.message_buffer_autoclear- ):- self.logger.warning(- f"Step usage ({self.last_step_usage.total_tokens} tokens) approaching "- f"context limit ({self.agent_state.llm_config.context_window}), triggering summarization."- )-- in_context_messages = await self.summarize_conversation_history(- in_context_messages=in_context_messages,- new_letta_messages=self.response_messages,- total_tokens=self.context_token_estimate,- force=True,- )-- # Clear to avoid duplication in next iteration- self.response_messages = []+ # refresh in-context messages (TODO: remove?)+ # in_context_messages = await self._refresh_messages(in_context_messages)if not self.should_continue:break- input_messages_to_persist = []-if i == max_steps - 1 and self.stop_reason is None:self.stop_reason = LettaStopReason(stop_reason=StopReasonType.max_steps.value)- # Rebuild context window after stepping (safety net)- if not self.agent_state.message_buffer_autoclear:- if self.context_token_estimate is not None:- await self.summarize_conversation_history(- in_context_messages=in_context_messages,- new_letta_messages=self.response_messages,- total_tokens=self.context_token_estimate,- force=False,- )- else:- self.logger.warning(- "Post-loop summarization skipped: last_step_usage is None. "- "No step completed successfully or usage stats were not updated."- )+ ## Rebuild context window after stepping (safety net)+ # if not self.agent_state.message_buffer_autoclear:+ # if self.context_token_estimate is not None:+ # await self.summarize_conversation_history(+ # in_context_messages=in_context_messages,+ # new_letta_messages=self.response_messages,+ # total_tokens=self.context_token_estimate,+ # force=False,+ # )+ # else:+ # self.logger.warning(+ # "Post-loop summarization skipped: last_step_usage is None. "+ # "No step completed successfully or usage stats were not updated."+ # )if self.stop_reason is None:self.stop_reason = LettaStopReason(stop_reason=StopReasonType.end_turn.value)@@ -400,10 +398,66 @@ class LettaAgentV3(LettaAgentV2):yield f"event: error\ndata: {error_message.model_dump_json()}\n\n"# Note: we don't send finish chunks here since we already errored+ async def _check_for_system_prompt_overflow(self, system_message):+ """+ Since the system prompt cannot be compacted, we need to check to see if it is the cause of the context overflow+ """+ system_prompt_token_estimate = await count_tokens(+ actor=self.actor,+ llm_config=self.agent_state.llm_config,+ messages=[system_message],+ )+ if system_prompt_token_estimate is not None and system_prompt_token_estimate >= self.agent_state.llm_config.context_window:+ self.should_continue = False+ self.stop_reason = LettaStopReason(stop_reason=StopReasonType.context_window_overflow_in_system_prompt.value)+ raise SystemPromptTokenExceededError(+ system_prompt_token_estimate=system_prompt_token_estimate,+ context_window=self.agent_state.llm_config.context_window,+ )++ async def _checkpoint_messages(self, run_id: str, step_id: str, new_messages: list[Message], in_context_messages: list[Message]):+ """+ Checkpoint the current message state - run this only when the current messages are 'safe' - meaning the step has completed successfully.++ This handles:+ - Persisting the new messages into the `messages` table+ - Updating the in-memory trackers for in-context messages (`self.in_context_messages`) and agent state (`self.agent_state.message_ids`)+ - Updating the DB with the current in-context messages (`self.agent_state.message_ids`)++ Args:+ run_id: The run ID to associate with the messages+ step_id: The step ID to associate with the messages+ new_messages: The new messages to persist+ in_context_messages: The current in-context messages+ """+ # make sure all the new messages have the correct run_id and step_id+ for message in new_messages:+ message.step_id = step_id+ message.run_id = run_id++ # persist the new message objects - ONLY place where messages are persisted+ persisted_messages = await self.message_manager.create_many_messages_async(+ new_messages,+ actor=self.actor,+ run_id=run_id,+ project_id=self.agent_state.project_id,+ template_id=self.agent_state.template_id,+ )++ # persist the in-context messages+ # TODO: somehow make sure all the message ids are already persisted+ await self.agent_manager.update_message_ids_async(+ agent_id=self.agent_state.id,+ message_ids=[m.id for m in in_context_messages],+ actor=self.actor,+ )+ self.agent_state.message_ids = [m.id for m in in_context_messages] # update in-memory state+ self.in_context_messages = in_context_messages # update in-memory state+@trace_methodasync def _step(self,- messages: list[Message],+ messages: list[Message], # current in-context messagesllm_adapter: LettaLLMAdapter,input_messages_to_persist: list[Message] | None = None,run_id: str | None = None,@@ -437,6 +491,8 @@ class LettaAgentV3(LettaAgentV2):if enforce_run_id_set and run_id is None:raise AssertionError("run_id is required when enforce_run_id_set is True")+ input_messages_to_persist = input_messages_to_persist or []+step_progression = StepProgression.START# TODO(@caren): clean this uptool_calls, content, agent_step_span, first_chunk, step_id, logged_step, step_start_ns, step_metrics = (@@ -464,13 +520,17 @@ class LettaAgentV3(LettaAgentV2):# Always refresh messages at the start of each step to pick up external inputs# (e.g., approval responses submitted by the client while this stream is running)try:+ # TODO: cleanup and de-dup+ # updates the system prompt with the latest blocks / message historiesmessages = await self._refresh_messages(messages)+except Exception as e:self.logger.warning(f"Failed to refresh messages at step start: {e}")approval_request, approval_response = _maybe_get_approval_messages(messages)tool_call_denials, tool_returns = [], []if approval_request and approval_response:+ # case of handling approval responsescontent = approval_request.content# Get tool calls that are pending@@ -541,6 +601,7 @@ class LettaAgentV3(LettaAgentV2):tool_return_truncation_chars=self._compute_tool_return_truncation_chars(),)# TODO: Extend to more providers, and also approval tool rules+ # TODO: this entire code block should be inside of the clients# Enable parallel tool use when no tool rules are attachedtry:no_tool_rules = (@@ -612,11 +673,25 @@ class LettaAgentV3(LettaAgentV2):except Exception as e:if isinstance(e, ContextWindowExceededError) and llm_request_attempt < summarizer_settings.max_summarizer_retries:# Retry case- messages = await self.summarize_conversation_history(- in_context_messages=messages,- new_letta_messages=self.response_messages,- force=True,+ summary_message, messages = await self.compact(+ messages, trigger_threshold=self.agent_state.llm_config.context_window)++ # checkpoint summarized messages+ # TODO: might want to delay this checkpoint in case of corrupated state+ try:+ await self._checkpoint_messages(+ run_id=run_id, step_id=step_id, new_messages=[summary_message], in_context_messages=messages+ )+ except SystemPromptTokenExceededError:+ self.stop_reason = LettaStopReason(+ stop_reason=StopReasonType.context_window_overflow_in_system_prompt.value+ )+ raise e+ except Exception as e:+ self.stop_reason = LettaStopReason(stop_reason=StopReasonType.error.value)+ self.logger.error(f"Unknown error occured for summarization run {run_id}: {e}")+ raise eelse:self.stop_reason = LettaStopReason(stop_reason=StopReasonType.error.value)self.logger.error(f"Unknown error occured for run {run_id}: {e}")@@ -637,8 +712,8 @@ class LettaAgentV3(LettaAgentV2):else:tool_calls = []- aggregated_persisted: list[Message] = []- persisted_messages, self.should_continue, self.stop_reason = await self._handle_ai_response(+ # get the new generated `Message` objects from handling the LLM response+ new_messages, self.should_continue, self.stop_reason = await self._handle_ai_response(tool_calls=tool_calls,valid_tool_names=[tool["name"] for tool in valid_tools],tool_rules_solver=self.tool_rules_solver,@@ -650,7 +725,7 @@ class LettaAgentV3(LettaAgentV2):content=content or llm_adapter.content,pre_computed_assistant_message_id=llm_adapter.message_id,step_id=step_id,- initial_messages=input_messages_to_persist,+ initial_messages=[], # input_messages_to_persist, # TODO: deprecate - super confusingagent_step_span=agent_step_span,is_final_step=(remaining_turns == 0),run_id=run_id,@@ -659,16 +734,26 @@ class LettaAgentV3(LettaAgentV2):tool_call_denials=tool_call_denials,tool_returns=tool_returns,)- aggregated_persisted.extend(persisted_messages)- # NOTE: there is an edge case where persisted_messages is empty (the LLM did a "no-op")- new_message_idx = len(input_messages_to_persist) if input_messages_to_persist else 0- self.response_messages.extend(aggregated_persisted[new_message_idx:])+ # extend trackers with new messages+ self.response_messages.extend(new_messages)+ messages.extend(new_messages)++ # step(...) has successfully completed! now we can persist messages and update the in-context messages + save metrics+ # persistence needs to happen before streaming to minimize chances of agent getting into an inconsistent state+ step_progression, step_metrics = await self._step_checkpoint_finish(step_metrics, agent_step_span, logged_step)+ await self._checkpoint_messages(+ run_id=run_id,+ step_id=step_id,+ new_messages=input_messages_to_persist + new_messages,+ in_context_messages=messages, # update the in-context messages+ )+ # yield back generated messagesif llm_adapter.supports_token_streaming():if tool_calls:# Stream each tool return if tools were executed- response_tool_returns = [msg for msg in aggregated_persisted if msg.role == "tool"]+ response_tool_returns = [msg for msg in new_messages if msg.role == "tool"]for tr in response_tool_returns:# Skip streaming for aggregated parallel tool returns (no per-call tool_call_id)if tr.tool_call_id is None and tr.tool_returns:@@ -677,7 +762,8 @@ class LettaAgentV3(LettaAgentV2):if include_return_message_types is None or tool_return_letta.message_type in include_return_message_types:yield tool_return_lettaelse:- filter_user_messages = [m for m in aggregated_persisted[new_message_idx:] if m.role != "user"]+ # TODO: modify this use step_response_messages+ filter_user_messages = [m for m in new_messages if m.role != "user"]letta_messages = Message.to_letta_messages_from_list(filter_user_messages,use_assistant_message=False, # NOTE: set to false@@ -689,11 +775,20 @@ class LettaAgentV3(LettaAgentV2):if include_return_message_types is None or message.message_type in include_return_message_types:yield message- # Note: message_ids update for approval responses now happens immediately after- # persistence in _handle_ai_response (line ~1093-1107) to prevent desync when- # the stream is interrupted and this generator is abandoned before being fully consumed- step_progression, step_metrics = await self._step_checkpoint_finish(step_metrics, agent_step_span, logged_step)+ # check compaction+ if self.context_token_estimate > self.agent_state.llm_config.context_window:+ summary_message, messages = await self.compact(messages, trigger_threshold=self.agent_state.llm_config.context_window)+ # TODO: persist + return the summary message+ # TODO: convert this to a SummaryMessage+ self.response_messages.append(summary_message)+ for message in Message.to_letta_messages(summary_message):+ yield message+ await self._checkpoint_messages(+ run_id=run_id, step_id=step_id, new_messages=[summary_message], in_context_messages=messages+ )+except Exception as e:+ # NOTE: message persistence does not happen in the case of an exception (rollback to previous state)self.logger.warning(f"Error during step processing: {e}")self.job_update_metadata = {"error": str(e)}@@ -707,20 +802,14 @@ class LettaAgentV3(LettaAgentV2):StopReasonType.invalid_tool_call,StopReasonType.invalid_llm_response,StopReasonType.llm_api_error,+ StopReasonType.context_window_overflow_in_system_prompt,):self.logger.warning("Error occurred during step processing, with unexpected stop reason: %s", self.stop_reason.stop_reason)raise efinally:+ # always make sure we update the step/run metadataself.logger.debug("Running cleanup for agent loop run: %s", run_id)self.logger.info("Running final update. Step Progression: %s", step_progression)-- # update message ids- message_ids = [m.id for m in messages]- await self.agent_manager.update_message_ids_async(- agent_id=self.agent_state.id,- message_ids=message_ids,- actor=self.actor,- )try:if step_progression == StepProgression.FINISHED:if not self.should_continue:@@ -728,7 +817,9 @@ class LettaAgentV3(LettaAgentV2):self.stop_reason = LettaStopReason(stop_reason=StopReasonType.end_turn.value)if logged_step and step_id:await self.step_manager.update_step_stop_reason(self.actor, step_id, self.stop_reason.stop_reason)- return+ if not self.stop_reason.stop_reason == StopReasonType.context_window_overflow_in_system_prompt:+ # only return if the stop reason is not context window overflow in system prompt+ returnif step_progression < StepProgression.STEP_LOGGED:# Error occurred before step was fully loggedimport traceback@@ -742,19 +833,6 @@ class LettaAgentV3(LettaAgentV2):error_traceback=traceback.format_exc(),stop_reason=self.stop_reason,)- if step_progression <= StepProgression.STREAM_RECEIVED:- if first_chunk and settings.track_errored_messages and input_messages_to_persist:- for message in input_messages_to_persist:- message.is_err = True- message.step_id = step_id- message.run_id = run_id- await self.message_manager.create_many_messages_async(- input_messages_to_persist,- actor=self.actor,- run_id=run_id,- project_id=self.agent_state.project_id,- template_id=self.agent_state.template_id,- )elif step_progression <= StepProgression.LOGGED_TRACE:if self.stop_reason is None:self.logger.warning("Error in step after logging step")@@ -806,6 +884,7 @@ class LettaAgentV3(LettaAgentV2):Unified approach: treats single and multi-tool calls uniformly to reduce code duplication."""+# 1. Handle no-tool cases (content-only or no-op)if not tool_calls and not tool_call_denials and not tool_returns:# Case 1a: No tool call, no content (LLM no-op)@@ -863,22 +942,7 @@ class LettaAgentV3(LettaAgentV2):add_heartbeat_on_continue=bool(heartbeat_reason),)messages_to_persist = (initial_messages or []) + assistant_message-- # Persist messages for no-tool cases- for message in messages_to_persist:- if message.run_id is None:- message.run_id = run_id- if message.step_id is None:- message.step_id = step_id-- persisted_messages = await self.message_manager.create_many_messages_async(- messages_to_persist,- actor=self.actor,- run_id=run_id,- project_id=self.agent_state.project_id,- template_id=self.agent_state.template_id,- )- return persisted_messages, continue_stepping, stop_reason+ return messages_to_persist, continue_stepping, stop_reason# 2. Check whether tool call requires approvalif not is_approval_response:@@ -896,21 +960,7 @@ class LettaAgentV3(LettaAgentV2):run_id=run_id,)messages_to_persist = (initial_messages or []) + approval_messages-- for message in messages_to_persist:- if message.run_id is None:- message.run_id = run_id- if message.step_id is None:- message.step_id = step_id-- persisted_messages = await self.message_manager.create_many_messages_async(- messages_to_persist,- actor=self.actor,- run_id=run_id,- project_id=self.agent_state.project_id,- template_id=self.agent_state.template_id,- )- return persisted_messages, False, LettaStopReason(stop_reason=StopReasonType.requires_approval.value)+ return messages_to_persist, False, LettaStopReason(stop_reason=StopReasonType.requires_approval.value)result_tool_returns = []@@ -1148,31 +1198,6 @@ class LettaAgentV3(LettaAgentV2):if message.step_id is None:message.step_id = step_id- # Persist all messages- persisted_messages = await self.message_manager.create_many_messages_async(- messages_to_persist,- actor=self.actor,- run_id=run_id,- project_id=self.agent_state.project_id,- template_id=self.agent_state.template_id,- )-- # Update message_ids immediately after persistence to prevent desync- # This handles approval responses where we need to keep message_ids in sync- if (- is_approval_response- and initial_messages- and len(initial_messages) == 1- and initial_messages[0].role == "approval"- and len(persisted_messages) >= 2- and persisted_messages[0].role == "approval"- and persisted_messages[1].role == "tool"- ):- self.agent_state.message_ids = self.agent_state.message_ids + [m.id for m in persisted_messages[:2]]- await self.agent_manager.update_message_ids_async(- agent_id=self.agent_state.id, message_ids=self.agent_state.message_ids, actor=self.actor- )-# 5g. Aggregate continuation decisionsaggregate_continue = any(persisted_continue_flags) if persisted_continue_flags else Falseaggregate_continue = aggregate_continue or tool_call_denials or tool_returns@@ -1193,7 +1218,7 @@ class LettaAgentV3(LettaAgentV2):# Force continuation for parallel tool executionaggregate_continue = Trueaggregate_stop_reason = None- return persisted_messages, aggregate_continue, aggregate_stop_reason+ return messages_to_persist, aggregate_continue, aggregate_stop_reason@trace_methoddef _decide_continuation(@@ -1282,178 +1307,118 @@ class LettaAgentV3(LettaAgentV2):return allowed_tools@trace_method- async def summarize_conversation_history(- self,- # The messages already in the context window- in_context_messages: list[Message],- # The messages produced by the agent in this step- new_letta_messages: list[Message],- # The token usage from the most recent LLM call (prompt + completion)- total_tokens: int | None = None,- # If force, then don't do any counting, just summarize- force: bool = False,- ) -> list[Message]:- trigger_summarization = force or (total_tokens and total_tokens > self.agent_state.llm_config.context_window)-- # no summarization if the last message is an approval request- latest_messages = in_context_messages + new_letta_messages- pending_approval = latest_messages[-1].role == "approval" and len(latest_messages[-1].tool_calls) > 0- if pending_approval:- trigger_summarization = False- self.logger.info(- f"trigger_summarization: {trigger_summarization}, total_tokens: {total_tokens}, context_window: {self.agent_state.llm_config.context_window}, pending_approval: {pending_approval}"- )- if not trigger_summarization:- # just update the message_ids- # TODO: gross to handle this here: we should move persistence elsewhere- new_in_context_messages = in_context_messages + new_letta_messages- message_ids = [m.id for m in new_in_context_messages]- await self.agent_manager.update_message_ids_async(- agent_id=self.agent_state.id,- message_ids=message_ids,- actor=self.actor,- )- self.agent_state.message_ids = message_ids- return new_in_context_messages-+ async def compact(self, messages, trigger_threshold: Optional[int] = None) -> Message:+ """+ Simplified compaction method. Does NOT do any persistence (handled in the loop)+ """+ # compact the current in-context messages (self.in_context_messages)# Use agent's summarizer_config if set, otherwise fall back to defaults# TODO: add this back# summarizer_config = self.agent_state.summarizer_config or get_default_summarizer_config(self.agent_state.llm_config)summarizer_config = get_default_summarizer_config(self.agent_state.llm_config._to_model_settings())-+ summarization_mode_used = summarizer_config.modeif summarizer_config.mode == "all":- summary_message_str, new_in_context_messages = await summarize_all(+ summary, compacted_messages = await summarize_all(actor=self.actor,llm_config=self.agent_state.llm_config,summarizer_config=summarizer_config,- in_context_messages=in_context_messages,- new_messages=new_letta_messages,+ in_context_messages=messages,)elif summarizer_config.mode == "sliding_window":try:- summary_message_str, new_in_context_messages = await summarize_via_sliding_window(+ summary, compacted_messages = await summarize_via_sliding_window(actor=self.actor,llm_config=self.agent_state.llm_config,summarizer_config=summarizer_config,- in_context_messages=in_context_messages,- new_messages=new_letta_messages,+ in_context_messages=messages,)except Exception as e:self.logger.error(f"Sliding window summarization failed with exception: {str(e)}. Falling back to all mode.")- summary_message_str, new_in_context_messages = await summarize_all(+ summary, compacted_messages = await summarize_all(actor=self.actor,llm_config=self.agent_state.llm_config,summarizer_config=summarizer_config,- in_context_messages=in_context_messages,- new_messages=new_letta_messages,+ in_context_messages=messages,)+ summarization_mode_used = "all"else:raise ValueError(f"Invalid summarizer mode: {summarizer_config.mode}")- # Persist the summary message to DB- summary_message_str_packed = package_summarize_message_no_counts(- summary=summary_message_str,- timezone=self.agent_state.timezone,- )- summary_message_obj = (- await convert_message_creates_to_messages(- message_creates=[- MessageCreate(- role=MessageRole.user,- content=[TextContent(text=summary_message_str_packed)],- )- ],- agent_id=self.agent_state.id,- timezone=self.agent_state.timezone,- # We already packed, don't pack again- wrap_user_message=False,- wrap_system_message=False,- run_id=None, # TODO: add this- )- )[0]- await self.message_manager.create_many_messages_async(- pydantic_msgs=[summary_message_obj],- actor=self.actor,- project_id=self.agent_state.project_id,- template_id=self.agent_state.template_id,- )-- # Update the message_ids in the agent state to include the summary- # plus whatever tail we decided to keep.- new_in_context_messages = [in_context_messages[0], summary_message_obj] + new_in_context_messages- new_in_context_message_ids = [m.id for m in new_in_context_messages]- await self.agent_manager.update_message_ids_async(- agent_id=self.agent_state.id,- message_ids=new_in_context_message_ids,- actor=self.actor,+ # update the token count+ self.context_token_estimate = await count_tokens(+ actor=self.actor, llm_config=self.agent_state.llm_config, messages=compacted_messages)- self.agent_state.message_ids = new_in_context_message_ids-- # After summarization, recompute an approximate token count for the- # updated in-context messages so that subsequent summarization- # decisions don't keep firing based on a stale, pre-summarization- # total_tokens value.- try:- new_total_tokens = await count_tokens(- actor=self.actor,- llm_config=self.agent_state.llm_config,- messages=new_in_context_messages,- )-- context_limit = self.agent_state.llm_config.context_window- trigger_threshold = int(context_limit * SUMMARIZATION_TRIGGER_MULTIPLIER)+ self.logger.info(f"Context token estimate after summarization: {self.context_token_estimate}")+ # if the trigger_threshold is provided, we need to make sure that the new token count is below it+ if trigger_threshold is not None and self.context_token_estimate >= trigger_threshold:# If even after summarization the context is still at or above# the proactive summarization threshold, treat this as a hard# failure: log loudly and evict all prior conversation state# (keeping only the system message) to avoid getting stuck in# repeated summarization loops.- if new_total_tokens > trigger_threshold:- self.logger.error(- "Summarization failed to sufficiently reduce context size: "- f"post-summarization tokens={new_total_tokens}, "- f"threshold={trigger_threshold}, context_window={context_limit}. "- "Evicting all prior messages without a summary to break potential loops.",+ self.logger.error(+ "Summarization failed to sufficiently reduce context size: "+ f"post-summarization tokens={self.context_token_estimate}, "+ f"threshold={trigger_threshold}, context_window={self.context_token_estimate}. "+ "Evicting all prior messages without a summary to break potential loops.",+ )++ # if we used the sliding window mode, try to summarize again with the all mode+ if summarization_mode_used == "sliding_window":+ # try to summarize again with the all mode+ summary, compacted_messages = await summarize_all(+ actor=self.actor,+ llm_config=self.agent_state.llm_config,+ summarizer_config=summarizer_config,+ in_context_messages=compacted_messages,)+ summarization_mode_used = "all"- # Keep only the system message in-context.- system_message = in_context_messages[0]- new_in_context_messages = [system_message]- new_in_context_message_ids = [system_message.id]+ self.context_token_estimate = await count_tokens(+ actor=self.actor, llm_config=self.agent_state.llm_config, messages=compacted_messages+ )- await self.agent_manager.update_message_ids_async(- agent_id=self.agent_state.id,- message_ids=new_in_context_message_ids,- actor=self.actor,+ # final edge case: the system prompt is the cause of the context overflow (raise error)+ if self.context_token_estimate >= trigger_threshold:+ await self._check_for_system_prompt_overflow(compacted_messages[0])++ # raise an error if this is STILL not the problem+ # do not throw an error, since we don't want to brick the agent+ self.logger.error(+ f"Failed to summarize messages after hard eviction and checking the system prompt token estimate: {self.context_token_estimate} > {trigger_threshold}"+ )+ else:+ self.logger.info(+ f"Summarization fallback succeeded in bringing the context size below the trigger threshold: {self.context_token_estimate} < {trigger_threshold}")- self.agent_state.message_ids = new_in_context_message_ids- # Recompute token usage for this minimal context and update- # context_token_estimate so future checks see the reduced size.- try:- minimal_tokens = await count_tokens(- actor=self.actor,- llm_config=self.agent_state.llm_config,- messages=new_in_context_messages,- )- self.context_token_estimate = minimal_tokens- except Exception as inner_e:- self.logger.warning(- f"Failed to recompute token usage after hard eviction: {inner_e}",- exc_info=True,- )+ # Persist the summary message to DB+ summary_message_str_packed = package_summarize_message_no_counts(+ summary=summary,+ timezone=self.agent_state.timezone,+ )+ summary_messages = await convert_message_creates_to_messages(+ message_creates=[+ MessageCreate(+ role=MessageRole.user,+ content=[TextContent(text=summary_message_str_packed)],+ )+ ],+ agent_id=self.agent_state.id,+ timezone=self.agent_state.timezone,+ # We already packed, don't pack again+ wrap_user_message=False,+ wrap_system_message=False,+ run_id=None, # TODO: add this+ )+ if not len(summary_messages) == 1:+ self.logger.error(f"Expected only one summary message, got {len(summary_messages)} in {summary_messages}")+ summary_message_obj = summary_messages[0]- return new_in_context_messages-- # Normal case: summarization succeeded in bringing us below the- # proactive threshold. Update context_token_estimate so future- # summarization checks reason over the *post*-summarization- # context size.- self.context_token_estimate = new_total_tokens- except Exception as e: # best-effort; never block the agent on this- self.logger.warning(- f"Failed to recompute token usage after summarization: {e}",- exc_info=True,- )+ # final messages: inject summarization message at the beginning+ final_messages = [compacted_messages[0]] + [summary_message_obj]+ if len(compacted_messages) > 1:+ final_messages += compacted_messages[1:]- return new_in_context_messages+ return summary_message_obj, final_messagesdiff --git a/letta/errors.py b/letta/errors.pyindex e99d1f5f4..3528596a9 100644--- a/letta/errors.py+++ b/letta/errors.py@@ -265,6 +265,16 @@ class ContextWindowExceededError(LettaError):)+class SystemPromptTokenExceededError(ContextWindowExceededError):+ """Error raised when the system prompt token estimate exceeds the context window."""++ def __init__(self, system_prompt_token_estimate: int, context_window: int):+ message = f"The system prompt tokens {system_prompt_token_estimate} exceeds the context window {context_window}. Please reduce the size of your system prompt, memory blocks, or increase the context window."+ super().__init__(+ message=message, details={"system_prompt_token_estimate": system_prompt_token_estimate, "context_window": context_window}+ )++class RateLimitExceededError(LettaError):"""Error raised when the llm rate limiter throttles api requests."""diff --git a/letta/schemas/letta_stop_reason.py b/letta/schemas/letta_stop_reason.pyindex d0f709153..fa67bc11c 100644--- a/letta/schemas/letta_stop_reason.py+++ b/letta/schemas/letta_stop_reason.py@@ -17,6 +17,7 @@ class StopReasonType(str, Enum):tool_rule = "tool_rule"cancelled = "cancelled"requires_approval = "requires_approval"+ context_window_overflow_in_system_prompt = "context_window_overflow_in_system_prompt"@propertydef run_status(self) -> RunStatus:@@ -33,6 +34,7 @@ class StopReasonType(str, Enum):StopReasonType.no_tool_call,StopReasonType.invalid_llm_response,StopReasonType.llm_api_error,+ StopReasonType.context_window_overflow_in_system_prompt,):return RunStatus.failedelif self == StopReasonType.cancelled:diff --git a/letta/server/rest_api/routers/v1/agents.py b/letta/server/rest_api/routers/v1/agents.pyindex cb751a0e4..08917d107 100644--- a/letta/server/rest_api/routers/v1/agents.py+++ b/letta/server/rest_api/routers/v1/agents.py@@ -2112,13 +2112,12 @@ async def summarize_messages(if agent_eligible and model_compatible:agent_loop = LettaAgentV3(agent_state=agent, actor=actor)in_context_messages = await server.message_manager.get_messages_by_ids_async(message_ids=agent.message_ids, actor=actor)- await agent_loop.summarize_conversation_history(- in_context_messages=in_context_messages,- new_letta_messages=[],- total_tokens=None,- force=True,+ summary_message, messages = await agent_loop.compact(+ messages=in_context_messages,)- # Summarization completed, return 204 No Content++ # update the agent state+ await agent_loop._checkpoint_messages(run_id=None, step_id=None, new_messages=[summary_message], in_context_messages=messages)else:raise HTTPException(status_code=status.HTTP_403_FORBIDDEN,diff --git a/letta/services/summarizer/summarizer_all.py b/letta/services/summarizer/summarizer_all.pyindex a27ae091a..a72258c1b 100644--- a/letta/services/summarizer/summarizer_all.py+++ b/letta/services/summarizer/summarizer_all.py@@ -20,7 +20,7 @@ async def summarize_all(# Actual summarization configurationsummarizer_config: SummarizerConfig,in_context_messages: List[Message],- new_messages: List[Message],+ # new_messages: List[Message],) -> str:"""Summarize the entire conversation history into a single summary.@@ -28,8 +28,7 @@ async def summarize_all(Returns:- The summary string"""- all_in_context_messages = in_context_messages + new_messages- messages_to_summarize = all_in_context_messages[1:]+ messages_to_summarize = in_context_messages[1:]# TODO: add fallback in case this has a context window errorsummary_message_str = await simple_summary(@@ -44,4 +43,4 @@ async def summarize_all(logger.warning(f"Summary length {len(summary_message_str)} exceeds clip length {summarizer_config.clip_chars}. Truncating.")summary_message_str = summary_message_str[: summarizer_config.clip_chars] + "... [summary truncated to fit]"- return summary_message_str, []+ return summary_message_str, [in_context_messages[0]]diff --git a/letta/services/summarizer/summarizer_sliding_window.py b/letta/services/summarizer/summarizer_sliding_window.pyindex 9f29f2066..f1f12dbef 100644--- a/letta/services/summarizer/summarizer_sliding_window.py+++ b/letta/services/summarizer/summarizer_sliding_window.py@@ -50,7 +50,7 @@ async def summarize_via_sliding_window(llm_config: LLMConfig,summarizer_config: SummarizerConfig,in_context_messages: List[Message],- new_messages: List[Message],+ # new_messages: List[Message],) -> Tuple[str, List[Message]]:"""If the total tokens is greater than the context window limit (or force=True),@@ -68,53 +68,42 @@ async def summarize_via_sliding_window(- The list of message IDs to keep in-context"""system_prompt = in_context_messages[0]- all_in_context_messages = in_context_messages + new_messages- total_message_count = len(all_in_context_messages)+ total_message_count = len(in_context_messages)# Starts at N% (eg 70%), and increments up until 100%message_count_cutoff_percent = max(1 - summarizer_config.sliding_window_percentage, 0.10) # Some arbitrary minimum value (10%) to avoid negatives from badly configured summarizer percentage- found_cutoff = False-- # Count tokens with system prompt, and message past cutoff point- assistant_message_index = None # Initialize to track if we found an assistant message- while not found_cutoff:- # Mark the approximate cutoff- message_cutoff_index = round(message_count_cutoff_percent * len(all_in_context_messages))-- # we've reached the maximum message cutoff- if message_cutoff_index >= total_message_count:+ assert summarizer_config.sliding_window_percentage <= 1.0, "Sliding window percentage must be less than or equal to 1.0"+ assistant_message_index = None+ approx_token_count = llm_config.context_window++ while (+ approx_token_count >= summarizer_config.sliding_window_percentage * llm_config.context_window and message_count_cutoff_percent < 1.0+ ):+ # calculate message_cutoff_index+ message_cutoff_index = round(message_count_cutoff_percent * total_message_count)++ # get index of first assistant message in range+ assistant_message_index = next(+ (i for i in range(message_cutoff_index, total_message_count) if in_context_messages[i].role == MessageRole.assistant), None+ )++ # if no assistant message in tail, break out of loop (since future iterations will continue hitting this case)+ if assistant_message_index is None:break- # Walk up the list until we find the first assistant message- for i in range(message_cutoff_index, total_message_count):- if all_in_context_messages[i].role == MessageRole.assistant:- assistant_message_index = i- break- else:- raise ValueError(f"No assistant message found from indices {message_cutoff_index} to {total_message_count}")-- # Count tokens of the hypothetical post-summarization buffer- post_summarization_buffer = [system_prompt] + all_in_context_messages[assistant_message_index:]- post_summarization_buffer_tokens = await count_tokens(actor, llm_config, post_summarization_buffer)-- # If hypothetical post-summarization count lower than the target remaining percentage?- if post_summarization_buffer_tokens <= summarizer_config.sliding_window_percentage * llm_config.context_window:- found_cutoff = True- else:- message_count_cutoff_percent += 0.10- if message_count_cutoff_percent >= 1.0:- message_cutoff_index = total_message_count- break-- # If we found the cutoff, summarize and return- # If we didn't find the cutoff and we hit 100%, this is equivalent to complete summarization+ # update token count+ post_summarization_buffer = [system_prompt] + in_context_messages[assistant_message_index:]+ approx_token_count = await count_tokens(actor, llm_config, post_summarization_buffer)++ # increment cutoff+ message_count_cutoff_percent += 0.10if assistant_message_index is None:raise ValueError("No assistant message found for sliding window summarization") # fall back to complete summarization- messages_to_summarize = all_in_context_messages[1:message_cutoff_index]+ messages_to_summarize = in_context_messages[1:assistant_message_index]summary_message_str = await simple_summary(messages=messages_to_summarize,@@ -128,5 +117,5 @@ async def summarize_via_sliding_window(logger.warning(f"Summary length {len(summary_message_str)} exceeds clip length {summarizer_config.clip_chars}. Truncating.")summary_message_str = summary_message_str[: summarizer_config.clip_chars] + "... [summary truncated to fit]"- updated_in_context_messages = all_in_context_messages[assistant_message_index:]- return summary_message_str, updated_in_context_messages+ updated_in_context_messages = in_context_messages[assistant_message_index:]+ return summary_message_str, [system_prompt] + updated_in_context_messagesdiff --git a/tests/integration_test_async_tool_sandbox.py b/tests/integration_test_async_tool_sandbox.pyindex 2d84bb397..172bd17f9 100644--- a/tests/integration_test_async_tool_sandbox.py+++ b/tests/integration_test_async_tool_sandbox.py@@ -908,7 +908,6 @@ async def test_e2b_sandbox_with_mixed_pip_requirements(check_e2b_key_is_set, too# Should succeed since both sandbox and tool pip requirements were installedassert "Success!" in result.func_return- assert "Status: 200" in result.func_returnassert "Array sum: 6" in result.func_returndiff --git a/tests/integration_test_summarizer.py b/tests/integration_test_summarizer.pyindex baa9fa662..3da6a3352 100644--- a/tests/integration_test_summarizer.py+++ b/tests/integration_test_summarizer.py@@ -16,13 +16,17 @@ import pytestfrom letta.agents.letta_agent_v2 import LettaAgentV2from letta.agents.letta_agent_v3 import LettaAgentV3from letta.config import LettaConfig-from letta.schemas.agent import CreateAgent+from letta.schemas.agent import CreateAgent, UpdateAgent+from letta.schemas.block import BlockUpdate, CreateBlockfrom letta.schemas.embedding_config import EmbeddingConfigfrom letta.schemas.enums import MessageRole+from letta.schemas.letta_message import LettaMessagefrom letta.schemas.letta_message_content import TextContent, ToolCallContent, ToolReturnContentfrom letta.schemas.llm_config import LLMConfig-from letta.schemas.message import Message as PydanticMessage+from letta.schemas.message import Message as PydanticMessage, MessageCreate+from letta.schemas.run import Run as PydanticRunfrom letta.server.server import SyncServer+from letta.services.run_manager import RunManager# ConstantsDEFAULT_EMBEDDING_CONFIG = EmbeddingConfig.default_config(provider="openai")@@ -40,8 +44,8 @@ def get_llm_config(filename: str, llm_config_dir: str = "tests/configs/llm_model# Test configurations - using a subset of models for summarization testsall_configs = ["openai-gpt-5-mini.json",- "claude-4-5-haiku.json",- "gemini-2.5-flash.json",+ # "claude-4-5-haiku.json",+ # "gemini-2.5-flash.json",# "gemini-2.5-flash-vertex.json", # Requires Vertex AI credentials# "openai-gpt-4.1.json",# "openai-o1.json",@@ -175,17 +179,12 @@ async def run_summarization(server: SyncServer, agent_state, in_context_messages2. Fetch messages via message_manager.get_messages_by_ids_async3. Call agent_loop.summarize_conversation_history with force=True"""- agent_loop = LettaAgentV2(agent_state=agent_state, actor=actor)+ agent_loop = LettaAgentV3(agent_state=agent_state, actor=actor)# Run summarization with force parameter- result = await agent_loop.summarize_conversation_history(- in_context_messages=in_context_messages,- new_letta_messages=[],- total_tokens=None,- force=force,- )+ summary_message, messages = await agent_loop.compact(messages=in_context_messages)- return result+ return summary_message, messages# ======================================================================================================================@@ -218,11 +217,24 @@ async def test_summarize_empty_message_buffer(server: SyncServer, actor, llm_con# Run summarization - this may fail with empty buffer, which is acceptable behaviortry:- result = await run_summarization(server, agent_state, in_context_messages, actor)+ summary, result = await run_summarization(server, agent_state, in_context_messages, actor)# If it succeeds, verify resultassert isinstance(result, list)- # With empty buffer, result should still be empty or contain only system messages- assert len(result) <= len(in_context_messages)++ # When summarization runs, V3 ensures that in-context messages follow+ # the pattern:+ # 1. System prompt+ # 2. User summary message (system_alert JSON)+ # 3. Remaining messages (which may be empty for this test)++ # We should always keep the original system message at the front.+ assert len(result) >= 1+ assert result[0].role == MessageRole.system++ # If summarization did in fact add a summary message, we expect it to+ # be the second message with user role.+ if len(result) >= 2:+ assert result[1].role == MessageRole.userexcept ValueError as e:# It's acceptable for summarization to fail on empty bufferassert "No assistant message found" in str(e) or "empty" in str(e).lower()@@ -255,7 +267,7 @@ async def test_summarize_initialization_messages_only(server: SyncServer, actor,# Run summarization - force=True with system messages only may failtry:- result = await run_summarization(server, agent_state, in_context_messages, actor, force=True)+ summary, result = await run_summarization(server, agent_state, in_context_messages, actor, force=True)# Verify resultassert isinstance(result, list)@@ -311,7 +323,7 @@ async def test_summarize_small_conversation(server: SyncServer, actor, llm_confi# Run summarization with force=True# Note: force=True with clear=True can be very aggressive and may fail on small message setstry:- result = await run_summarization(server, agent_state, in_context_messages, actor, force=True)+ summary, result = await run_summarization(server, agent_state, in_context_messages, actor, force=True)# Verify resultassert isinstance(result, list)@@ -404,7 +416,7 @@ async def test_summarize_large_tool_calls(server: SyncServer, actor, llm_config:assert total_content_size > 40000, f"Expected large messages, got {total_content_size} chars"# Run summarization- result = await run_summarization(server, agent_state, in_context_messages, actor)+ summary, result = await run_summarization(server, agent_state, in_context_messages, actor)# Verify resultassert isinstance(result, list)@@ -508,7 +520,7 @@ async def test_summarize_multiple_large_tool_calls(server: SyncServer, actor, llassert total_content_size > 40000, f"Expected large messages, got {total_content_size} chars"# Run summarization- result = await run_summarization(server, agent_state, in_context_messages, actor)+ summary, result = await run_summarization(server, agent_state, in_context_messages, actor)# Verify resultassert isinstance(result, list)@@ -579,7 +591,7 @@ async def test_summarize_truncates_large_tool_return(server: SyncServer, actor,assert original_size > 90000, f"Expected tool return >90k chars, got {original_size}"# Run summarization- result = await run_summarization(server, agent_state, in_context_messages, actor)+ summary, result = await run_summarization(server, agent_state, in_context_messages, actor)# Verify resultassert isinstance(result, list)@@ -678,12 +690,7 @@ async def test_summarize_with_mode(server: SyncServer, actor, llm_config: LLMConwith patch("letta.agents.letta_agent_v3.get_default_summarizer_config", mock_get_default_summarizer_config):agent_loop = LettaAgentV3(agent_state=agent_state, actor=actor)- result = await agent_loop.summarize_conversation_history(- in_context_messages=in_context_messages,- new_letta_messages=new_letta_messages,- total_tokens=None,- force=True,- )+ summary, result = await agent_loop.compact(messages=in_context_messages)assert isinstance(result, list)@@ -700,24 +707,21 @@ async def test_summarize_with_mode(server: SyncServer, actor, llm_config: LLMConprint()if mode == "all":- # For "all" mode, result should be just the summary message- assert len(result) == 2, f"Expected 1 message for 'all' mode, got {len(result)}"+ # For "all" mode, V3 keeps:+ # 1. System prompt+ # 2. A single user summary message (system_alert JSON)+ # and no remaining historical messages.+ assert len(result) == 2, f"Expected 2 messages for 'all' mode (system + summary), got {len(result)}"+ assert result[0].role == MessageRole.system+ assert result[1].role == MessageRole.userelse:- # For "sliding_window" mode, result should include recent messages + summary- assert len(result) > 1, f"Expected >1 messages for 'sliding_window' mode, got {len(result)}"- # validate new user message- assert result[-1].role == MessageRole.user and result[-1].agent_id == agent_state.id, (- f"Expected new user message with agent_id {agent_state.id}, got {result[-1]}"- )- assert "This is a new user message" in result[-1].content[0].text, (- f"Expected 'This is a new user message' in the user message, got {result[-1]}"- )-- # validate system message- assert result[0].role == MessageRole.system- # validate summary message- assert "prior messages" in result[1].content[0].text, f"Expected 'prior messages' in the summary message, got {result[1]}"- print(f"Mode '{mode}' with {llm_config.model}: {len(in_context_messages)} -> {len(result)} messages")+ # For "sliding_window" mode, result should include:+ # 1. System prompt+ # 2. User summary message+ # 3+. Recent user/assistant messages inside the window.+ assert len(result) > 2, f"Expected >2 messages for 'sliding_window' mode, got {len(result)}"+ assert result[0].role == MessageRole.system+ assert result[1].role == MessageRole.user@pytest.mark.asyncio@@ -740,15 +744,16 @@ async def test_v3_summarize_hard_eviction_when_still_over_threshold(is still above the trigger threshold.3. We verify that LettaAgentV3:- Logs an error about summarization failing to reduce context size.- - Evicts all prior messages, keeping only the system message.+ - Evicts all prior messages, keeping only the system message plus a+ single synthetic user summary message (system_alert).- Updates `context_token_estimate` to the token count of the minimalcontext so future steps don't keep re-triggering summarization basedon a stale, oversized value."""# Build a small but non-trivial conversation with an explicit system- # message so that after hard eviction we expect to keep exactly that one- # message.+ # message so that after hard eviction we expect to keep exactly that+ # system message plus a single user summary message.messages = [PydanticMessage(role=MessageRole.system,@@ -766,6 +771,10 @@ async def test_v3_summarize_hard_eviction_when_still_over_threshold(agent_state, in_context_messages = await create_agent_with_messages(server, actor, llm_config, messages)+ print("ORIGINAL IN-CONTEXT MESSAGES ======")+ for msg in in_context_messages:+ print(f"MSG: {msg}")+# Create the V3 agent loopagent_loop = LettaAgentV3(agent_state=agent_state, actor=actor)@@ -787,36 +796,26 @@ async def test_v3_summarize_hard_eviction_when_still_over_threshold(caplog.set_level("ERROR")- result = await agent_loop.summarize_conversation_history(- in_context_messages=in_context_messages,- new_letta_messages=[],- # total_tokens is not used when force=True for triggering, but we- # set it to a large value for clarity.- total_tokens=llm_config.context_window * 2 if llm_config.context_window else None,- force=True,+ summary, result = await agent_loop.compact(+ messages=in_context_messages,+ trigger_threshold=context_limit,)# We should have made exactly two token-count calls: one for the# summarized context, one for the hard-evicted minimal context.assert mock_count_tokens.call_count == 2- # After hard eviction, only the system message should remain in-context.+ print("COMPACTED RESULT ======")+ for msg in result:+ print(f"MSG: {msg}")++ # After hard eviction, we keep only:+ # 1. The system prompt+ # 2. The synthetic user summary message.assert isinstance(result, list)- assert len(result) == 1, f"Expected only the system message after hard eviction, got {len(result)} messages"+ assert len(result) == 2, f"Expected system + summary after hard eviction, got {len(result)} messages"assert result[0].role == MessageRole.system-- # Agent state should also reflect exactly one message id.- assert len(agent_loop.agent_state.message_ids) == 1-- # context_token_estimate should be updated to the minimal token count- # (second side-effect value from count_tokens), rather than the original- # huge value.- assert agent_loop.context_token_estimate == 10-- # Verify that we logged an error about summarization failing to reduce- # context size.- error_logs = [rec for rec in caplog.records if "Summarization failed to sufficiently reduce context size" in rec.getMessage()]- assert error_logs, "Expected an error log when summarization fails to reduce context size sufficiently"+ assert result[1].role == MessageRole.user# ======================================================================================================================@@ -893,7 +892,6 @@ async def test_sliding_window_cutoff_index_does_not_exceed_message_count(server:llm_config=llm_config,summarizer_config=summarizer_config,in_context_messages=messages,- new_messages=[],)# Verify the summary was generated (actual LLM response)@@ -924,6 +922,105 @@ async def test_sliding_window_cutoff_index_does_not_exceed_message_count(server:raise+ "llm_config",+ TESTED_LLM_CONFIGS,+ ids=[c.model for c in TESTED_LLM_CONFIGS],+)+async def test_large_system_prompt_summarization(server: SyncServer, actor, llm_config: LLMConfig):+ """+ Test edge case of large system prompt / memory blocks.++ This test verifies that summarization handles the case where the system prompt+ and memory blocks are very large, potentially consuming most of the context window.+ The summarizer should gracefully handle this scenario without errors.+ """++ # Override context window to be small so we trigger summarization+ llm_config.context_window = 10000++ # Create agent with large system prompt and memory blocks+ agent_name = f"test_agent_large_system_prompt_{llm_config.model}".replace(".", "_").replace("/", "_")+ agent_create = CreateAgent(+ name=agent_name,+ llm_config=llm_config,+ embedding_config=DEFAULT_EMBEDDING_CONFIG,+ system="SYSTEM PROMPT " * 10000, # Large system prompt+ memory_blocks=[+ CreateBlock(+ label="human",+ limit=200000,+ value="NAME " * 10000, # Large memory block+ )+ ],+ )+ agent_state = await server.agent_manager.create_agent_async(agent_create, actor=actor)++ # Create a run for the agent using RunManager+ run = PydanticRun(agent_id=agent_state.id)+ run = await RunManager().create_run(pydantic_run=run, actor=actor)++ # Create the agent loop using LettaAgentV3+ agent_loop = LettaAgentV3(agent_state=agent_state, actor=actor)++ # message the agent+ input_message = MessageCreate(role=MessageRole.user, content="Hello")++ # Call step on the agent - may trigger summarization due to large context+ from letta.errors import SystemPromptTokenExceededError++ with pytest.raises(SystemPromptTokenExceededError):+ response = await agent_loop.step(+ input_messages=[input_message],+ run_id=run.id,+ max_steps=3,+ )++ # Repair the agent by shortening the memory blocks and system prompt+ # Update system prompt to a shorter version+ short_system_prompt = "You are a helpful assistant."+ await server.agent_manager.update_agent_async(+ agent_id=agent_state.id,+ agent_update=UpdateAgent(system=short_system_prompt),+ actor=actor,+ )++ # Update memory block to a shorter version+ short_memory_value = "The user's name is Alice."+ await server.agent_manager.modify_block_by_label_async(+ agent_id=agent_state.id,+ block_label="human",+ block_update=BlockUpdate(value=short_memory_value),+ actor=actor,+ )++ # Reload agent state after repairs+ agent_state = await server.agent_manager.get_agent_by_id_async(agent_id=agent_state.id, actor=actor)+ print("REPAIRED AGENT STATE ======")+ print(agent_state.system)+ print(agent_state.blocks)++ # Create a new run for the repaired agent+ run = PydanticRun(agent_id=agent_state.id)+ run = await RunManager().create_run(pydantic_run=run, actor=actor)++ # Create a new agent loop with the repaired agent state+ agent_loop = LettaAgentV3(agent_state=agent_state, actor=actor)++ # Now the agent should be able to respond without context window errors+ response = await agent_loop.step(+ input_messages=[input_message],+ run_id=run.id,+ max_steps=3,+ )++ # Verify we got a valid response after repair+ assert response is not None+ assert response.messages is not None+ print(f"Agent successfully responded after repair with {len(response.messages)} messages")++# @pytest.mark.asyncio# async def test_context_window_overflow_triggers_summarization_in_streaming(server: SyncServer, actor):# """@@ -1342,11 +1439,10 @@ async def test_summarize_all(server: SyncServer, actor, llm_config: LLMConfig):llm_config=llm_config,summarizer_config=summarizer_config,in_context_messages=messages,- new_messages=[],)# Verify the summary was generated- assert len(new_in_context_messages) == 0+ assert len(new_in_context_messages) == 1assert summary is not Noneassert len(summary) > 0assert len(summary) <= 2000