Platform for stateful agents: AI with advanced memory that can learn and self-improve over time.
feat: modify summarization for auto mode (#9874)
cthomas0ca33c58005b
diff --git a/letta/agents/letta_agent_v3.py b/letta/agents/letta_agent_v3.pyindex e595b49a6..bc6cc239e 100644--- a/letta/agents/letta_agent_v3.py+++ b/letta/agents/letta_agent_v3.py@@ -961,7 +961,7 @@ class LettaAgentV3(LettaAgentV2):actor=self.actor,)if not is_primary:- self.logger.info(f"[AUTO MODE]: primary {primary_handle} rerouted, falling back to {active_llm_config.handle}")+ self.logger.info(f"[LLM ROUTER]: primary {primary_handle} rerouted, falling back to {active_llm_config.handle}")active_llm_client = LLMClient.create(provider_type=active_llm_config.model_endpoint_type,put_inner_thoughts_first=True,@@ -1048,10 +1048,10 @@ class LettaAgentV3(LettaAgentV2):if llm_adapter.supports_token_streaming():if include_return_message_types is None or chunk.message_type in include_return_message_types:yield chunk- # Report success to circuit breaker- if is_auto_mode and is_primary:- routing_client = await get_llm_routing_client()- await routing_client.record_success(primary_handle)+ # Report success to circuit breaker (only for models with fallback routes)+ routing_client = await get_llm_routing_client()+ if routing_client.get_fallback_handle(active_llm_config.handle):+ await routing_client.record_success(active_llm_config.handle)# If you've reached this point without an error, break out of retry loopbreakexcept ValueError as e:@@ -1061,59 +1061,34 @@ class LettaAgentV3(LettaAgentV2):self.stop_reason = LettaStopReason(stop_reason=StopReasonType.invalid_llm_response.value)raise eexcept (LLMRateLimitError, LLMServerError, LLMProviderOverloaded) as e:- if is_auto_mode and is_primary:- routing_client = await get_llm_routing_client()- await routing_client.record_failure(primary_handle)+ # Check if there's a fallback route for the current model+ routing_client = await get_llm_routing_client()+ current_handle = active_llm_config.handle+ fallback_handle = routing_client.get_fallback_handle(current_handle)- fallback_config = await routing_client.get_fallback_config(+ if fallback_handle:+ await routing_client.record_failure(current_handle)++ fallback_config = await routing_client.get_fallback_config_for_handle(+ fallback_handle=fallback_handle,stored_llm_config=self.agent_state.llm_config,actor=self.actor,)self.logger.warning(- f"[AUTO MODE]: primary {primary_handle} failed ({type(e).__name__}), falling back to {fallback_config.handle}"+ f"[LLM ROUTER]: {current_handle} failed ({type(e).__name__}), falling back to {fallback_config.handle}")- fallback_client = LLMClient.create(++ # Switch to fallback for this attempt and any subsequent retries (e.g. compaction)+ active_llm_config = fallback_config+ active_llm_client = LLMClient.create(provider_type=fallback_config.model_endpoint_type,put_inner_thoughts_first=True,actor=self.actor,)- fallback_request_data = fallback_client.build_request_data(- agent_type=self.agent_state.agent_type,- messages=messages,- llm_config=fallback_config,- tools=valid_tools,- force_tool_call=force_tool_call,- requires_subsequent_tool_call=self._require_tool_call,- tool_return_truncation_chars=self._compute_tool_return_truncation_chars(),- )- fallback_adapter = SimpleLLMRequestAdapter(- llm_client=fallback_client,- llm_config=fallback_config,- call_type=LLMCallType.agent_step,- agent_id=self.agent_state.id,- agent_tags=self.agent_state.tags,- run_id=run_id,- org_id=self.actor.organization_id,- user_id=self.actor.id,- )- fallback_invocation = fallback_adapter.invoke_llm(- request_data=fallback_request_data,- messages=messages,- tools=valid_tools,- use_assistant_message=False,- requires_approval_tools=self.tool_rules_solver.get_requires_approval_tools(- set([t["name"] for t in valid_tools])- )- + [ct.name for ct in self.client_tools],- step_id=step_id,- actor=self.actor,- )- async for chunk in fallback_invocation:- if fallback_adapter.supports_token_streaming():- if include_return_message_types is None or chunk.message_type in include_return_message_types:- yield chunk- llm_adapter = fallback_adapter- break+ llm_adapter.llm_client = active_llm_client+ llm_adapter.llm_config = active_llm_config+ is_primary = False+ continueself.stop_reason = LettaStopReason(stop_reason=StopReasonType.llm_api_error.value)raise eexcept LLMError as e:diff --git a/letta/services/llm_router/__init__.py b/letta/services/llm_router/__init__.pyindex 813f6f93e..521cf0ea2 100644--- a/letta/services/llm_router/__init__.py+++ b/letta/services/llm_router/__init__.py@@ -1,12 +1,12 @@-"""LLM routing client with circuit breaker and auto mode support.+"""LLM routing client with circuit breaker and fallback support.Conditionally imports Redis-backed implementation or falls back to noop base."""try:- from .client import LLMRoutingClient, get_llm_routing_client+ from .llm_router_client import LLMRoutingClient, get_llm_routing_clientexcept ImportError:- from .client_base import LLMRoutingClient+ from .llm_router_client_base import LLMRoutingClientasync def get_llm_routing_client():return LLMRoutingClient()diff --git a/letta/services/llm_router/client.py b/letta/services/llm_router/client.pydeleted file mode 100644index 392b3fccd..000000000--- a/letta/services/llm_router/client.py+++ /dev/null@@ -1,247 +0,0 @@-"""Redis-backed LLM routing client for model health tracking and auto mode resolution.--Tracks overload errors per model handle using Redis sorted sets for-sliding window error counting. Works across multiple pods.--The noop base class lives in client_base.py.-"""--import time-from enum import Enum-from typing import TYPE_CHECKING, Optional--from letta.data_sources.redis_client import AsyncRedisClient-from letta.log import get_logger--from .client_base import LLMRoutingClient as LLMRoutingClientBase--if TYPE_CHECKING:- from letta.schemas.llm_config import LLMConfig- from letta.schemas.user import User--logger = get_logger(__name__)--# Auto mode model routing-AUTO_MODE_PRIMARY = "baseten/glm-5"-AUTO_MODE_FALLBACK = "zai/glm-5"--# Circuit breaker configuration-ERROR_WINDOW_SECONDS = 60 # Sliding window for error counting-ERROR_THRESHOLD = 1 # Errors needed to trip circuit-OPEN_DURATION_SECONDS = 30 # How long circuit stays open--# Redis key prefixes-CIRCUIT_ERRORS_KEY = "circuit:errors:"-CIRCUIT_STATE_KEY = "circuit:state:"-CIRCUIT_OPENED_AT_KEY = "circuit:opened:"---def _build_baseten_config() -> "LLMConfig":- """Build a hardcoded LLMConfig for Baseten GLM-5 (not discoverable via provider sync)."""- from letta.schemas.enums import ProviderCategory- from letta.schemas.llm_config import LLMConfig-- return LLMConfig(- model="zai-org/GLM-5",- model_endpoint_type="baseten",- model_endpoint="3m5zok9w",- context_window=180000,- max_tokens=16384,- handle="baseten/glm-5",- provider_name="baseten",- provider_category=ProviderCategory.base,- )---def _build_fireworks_config() -> "LLMConfig":- """Build a hardcoded LLMConfig for Fireworks GLM-5 (not discoverable via provider sync)."""- from letta.schemas.enums import ProviderCategory- from letta.schemas.llm_config import LLMConfig- from letta.settings import model_settings-- return LLMConfig(- model="accounts/fireworks/models/glm-5",- model_endpoint_type="fireworks",- model_endpoint=model_settings.fireworks_api_base,- context_window=180000,- max_tokens=16384,- handle="fireworks/glm-5",- provider_name="fireworks",- provider_category=ProviderCategory.base,- )---class CircuitState(str, Enum):- CLOSED = "CLOSED"- OPEN = "OPEN"---class LLMRoutingClient(LLMRoutingClientBase):- """Redis-backed LLM routing client for model health tracking and auto mode routing.-- Uses an internal circuit breaker (Redis sorted sets) for sliding window error- counting across pods. The circuit breaker is an implementation detail.- """-- def __init__(self, redis_client: AsyncRedisClient):- self.redis = redis_client-- async def resolve_auto_mode_config(- self,- stored_llm_config: "LLMConfig",- actor: "User",- ) -> tuple["LLMConfig", bool, str]:- """Resolve an auto mode handle to an actual model config.-- Uses the internal circuit breaker to decide between primary and fallback.- Preserves user-configured context_window and max_tokens from the stored config.-- Args:- stored_llm_config: The agent's stored LLM config (with auto mode handle).- actor: The user actor for provider lookups.-- Returns:- Tuple of (resolved_config, is_primary, primary_handle).- """- from letta.services.provider_manager import ProviderManager- from letta.settings import model_settings-- provider_manager = ProviderManager()- primary_handle = AUTO_MODE_PRIMARY-- if not model_settings.auto_mode_enabled:- logger.info(f"[AUTO MODE]: primary {primary_handle} disabled via kill switch, using {AUTO_MODE_FALLBACK}")- elif await self._is_healthy(primary_handle):- config = _build_baseten_config()- config = config.model_copy(- update={- "context_window": min(stored_llm_config.context_window, config.context_window),- "max_tokens": min(stored_llm_config.max_tokens, config.max_tokens),- }- )- return config, True, primary_handle- else:- logger.warning(f"[AUTO MODE]: primary {primary_handle} unhealthy, using {AUTO_MODE_FALLBACK}")-- config = await provider_manager.get_llm_config_from_handle(AUTO_MODE_FALLBACK, actor)- config = config.model_copy(- update={- "context_window": min(stored_llm_config.context_window, config.context_window),- "max_tokens": min(stored_llm_config.max_tokens, config.max_tokens),- }- )- return config, False, primary_handle-- async def record_failure(self, handle: str) -> None:- """Record a failure for a model handle. May trip the circuit.-- Args:- handle: The model handle that failed.- """- client = await self.redis.get_client()- now = time.time()- errors_key = f"{CIRCUIT_ERRORS_KEY}{handle}"-- await client.zadd(errors_key, {str(now): now})- await client.zremrangebyscore(errors_key, 0, now - ERROR_WINDOW_SECONDS)- await client.expire(errors_key, ERROR_WINDOW_SECONDS * 2)-- error_count = await client.zcard(errors_key)- if error_count >= ERROR_THRESHOLD:- await self._open_circuit(client, handle)-- async def record_success(self, handle: str) -> None:- """Record a success for a model handle. Closes circuit if open.-- Args:- handle: The model handle that succeeded.- """- state = await self._get_state(handle)- if state == CircuitState.OPEN:- await self._close_circuit(handle)-- async def get_fallback_config(- self,- stored_llm_config: "LLMConfig",- actor: "User",- ) -> "LLMConfig":- """Get the fallback config for auto mode.-- Args:- stored_llm_config: The agent's stored LLM config (with auto mode handle).- actor: The user actor for provider lookups.-- Returns:- The fallback LLM config with preserved context_window and max_tokens.- """- from letta.services.provider_manager import ProviderManager-- provider_manager = ProviderManager()- fallback_config = await provider_manager.get_llm_config_from_handle(AUTO_MODE_FALLBACK, actor)- fallback_config = fallback_config.model_copy(- update={- "context_window": min(stored_llm_config.context_window, fallback_config.context_window),- "max_tokens": min(stored_llm_config.max_tokens, fallback_config.max_tokens),- }- )- return fallback_config-- async def _is_healthy(self, handle: str) -> bool:- """Check if a model handle is healthy (internal circuit breaker).-- Returns True if the circuit is closed, or if the open duration has elapsed- (allowing a retry probe).-- Args:- handle: The model handle to check.-- Returns:- True if the model should be tried, False if it should be skipped.- """- state = await self._get_state(handle)- if state == CircuitState.CLOSED:- return True-- opened_at = await self._get_opened_at(handle)- if opened_at and time.time() - opened_at >= OPEN_DURATION_SECONDS:- return True-- return False-- async def _get_state(self, handle: str) -> CircuitState:- client = await self.redis.get_client()- state = await client.get(f"{CIRCUIT_STATE_KEY}{handle}")- return CircuitState.OPEN if state == CircuitState.OPEN.value else CircuitState.CLOSED-- async def _get_opened_at(self, handle: str) -> Optional[float]:- client = await self.redis.get_client()- val = await client.get(f"{CIRCUIT_OPENED_AT_KEY}{handle}")- return float(val) if val else None-- async def _open_circuit(self, client, handle: str) -> None:- now = time.time()- await client.set(f"{CIRCUIT_STATE_KEY}{handle}", CircuitState.OPEN.value, ex=OPEN_DURATION_SECONDS * 2)- await client.set(f"{CIRCUIT_OPENED_AT_KEY}{handle}", str(now), ex=OPEN_DURATION_SECONDS * 2)- logger.warning(f"Circuit OPENED for model {handle}")-- async def _close_circuit(self, handle: str) -> None:- client = await self.redis.get_client()- await client.delete(f"{CIRCUIT_STATE_KEY}{handle}", f"{CIRCUIT_OPENED_AT_KEY}{handle}", f"{CIRCUIT_ERRORS_KEY}{handle}")- logger.info(f"Circuit CLOSED for model {handle}")---async def get_llm_routing_client() -> LLMRoutingClientBase:- """Factory that returns the appropriate LLM routing client instance.-- Returns the Redis-backed implementation if Redis is available,- otherwise returns the noop base.-- Returns:- An LLMRoutingClient instance.- """- from letta.data_sources.redis_client import NoopAsyncRedisClient, get_redis_client-- redis_client = await get_redis_client()- if isinstance(redis_client, NoopAsyncRedisClient):- return LLMRoutingClientBase()- return LLMRoutingClient(redis_client)diff --git a/letta/services/llm_router/client_base.py b/letta/services/llm_router/llm_router_client_base.pysimilarity index 65%rename from letta/services/llm_router/client_base.pyrename to letta/services/llm_router/llm_router_client_base.pyindex 3bb9f8159..bce6f69a9 100644--- a/letta/services/llm_router/client_base.py+++ b/letta/services/llm_router/llm_router_client_base.py@@ -1,10 +1,10 @@"""Base LLM routing client.This is the base (OSS/self-hosted) implementation without Redis support.-The Redis-backed implementation lives in client.py.+The Redis-backed implementation lives in llm_router_client.py."""-from typing import TYPE_CHECKING+from typing import TYPE_CHECKING, Optionalif TYPE_CHECKING:from letta.schemas.llm_config import LLMConfig@@ -56,12 +56,38 @@ class LLMRoutingClient:stored_llm_config: The agent's stored LLM config.actor: The user actor for provider lookups.- Returns:- The fallback LLM config.-Raises:RuntimeError: Auto mode requires Redis for circuit breaker support."""raise RuntimeError("Auto mode requires Redis for circuit breaker support. Configure Redis or disable auto_mode_enabled in settings.")++ async def get_fallback_config_for_handle(+ self,+ fallback_handle: str,+ stored_llm_config: "LLMConfig",+ actor: "User",+ ) -> "LLMConfig":+ """Get a fallback config for any handle (noop in base).++ Args:+ fallback_handle: The fallback model handle to resolve.+ stored_llm_config: The agent's stored LLM config.+ actor: The user actor for provider lookups.++ Raises:+ RuntimeError: Fallback routing requires Redis for circuit breaker support.+ """+ raise RuntimeError("Fallback routing requires Redis for circuit breaker support.")++ def get_fallback_handle(self, handle: str) -> Optional[str]:+ """Get the fallback handle for a given primary handle (noop in base).++ Args:+ handle: The primary model handle.++ Returns:+ None — no fallback routes configured without Redis.+ """+ return Nonediff --git a/letta/services/summarizer/compact.py b/letta/services/summarizer/compact.pyindex 68c721b8f..8c4d0ee61 100644--- a/letta/services/summarizer/compact.py+++ b/letta/services/summarizer/compact.py@@ -49,6 +49,9 @@ async def build_summarizer_llm_config(then apply any explicit ``compaction_settings.model_settings`` via``_to_legacy_config_params``.+ For auto mode agents (letta/baseten providers), uses the LLM router's+ circuit breaker to try the primary model first, falling back if unhealthy.+Args:agent_llm_config: The agent's LLM configuration to use as base.summarizer_config: Compaction settings with optional model override.@@ -59,6 +62,20 @@ async def build_summarizer_llm_config("""from letta.schemas.enums import ProviderType+ # Auto mode agents: use the LLM router to resolve primary vs fallback+ if not summarizer_config.model and agent_llm_config.provider_name in ("letta", "baseten"):+ try:+ from letta.services.llm_router import get_llm_routing_client++ routing_client = await get_llm_routing_client()+ resolved_config, _, _ = await routing_client.resolve_auto_mode_config(+ stored_llm_config=agent_llm_config,+ actor=actor,+ )+ return resolved_config+ except Exception as e:+ logger.warning(f"Failed to resolve auto mode config for summarizer: {e}. Falling back to defaults.")+# If no summarizer model specified, use lightweight provider-specific defaultsif not summarizer_config.model:provider_name = agent_llm_config.provider_name or agent_llm_config.model_endpoint_typediff --git a/letta/services/summarizer/summarizer_config.py b/letta/services/summarizer/summarizer_config.pyindex a1cb46a66..bbfe7f631 100644--- a/letta/services/summarizer/summarizer_config.py+++ b/letta/services/summarizer/summarizer_config.py@@ -14,6 +14,7 @@ def get_default_summarizer_model(provider_type: ProviderType) -> str | None:ProviderType.anthropic: "anthropic/claude-haiku-4-5",ProviderType.openai: "openai/gpt-5-mini",ProviderType.google_ai: "google_ai/gemini-2.5-flash",+ ProviderType.letta: "zai/glm-5",}return summarizer_defaults.get(provider_type)