Skip to content

armgit / letta

Public Imported from GitHub
0

Platform for stateful agents: AI with advanced memory that can learn and self-improve over time.

feat: modify summarization for auto mode (#9874)

cthomas0ca33c58005b
diff --git a/letta/agents/letta_agent_v3.py b/letta/agents/letta_agent_v3.py
index e595b49a6..bc6cc239e 100644
--- a/letta/agents/letta_agent_v3.py
+++ b/letta/agents/letta_agent_v3.py
@@ -961,7 +961,7 @@ class LettaAgentV3(LettaAgentV2):
actor=self.actor,
)
if not is_primary:
- self.logger.info(f"[AUTO MODE]: primary {primary_handle} rerouted, falling back to {active_llm_config.handle}")
+ self.logger.info(f"[LLM ROUTER]: primary {primary_handle} rerouted, falling back to {active_llm_config.handle}")
active_llm_client = LLMClient.create(
provider_type=active_llm_config.model_endpoint_type,
put_inner_thoughts_first=True,
@@ -1048,10 +1048,10 @@ class LettaAgentV3(LettaAgentV2):
if llm_adapter.supports_token_streaming():
if include_return_message_types is None or chunk.message_type in include_return_message_types:
yield chunk
- # Report success to circuit breaker
- if is_auto_mode and is_primary:
- routing_client = await get_llm_routing_client()
- await routing_client.record_success(primary_handle)
+ # Report success to circuit breaker (only for models with fallback routes)
+ routing_client = await get_llm_routing_client()
+ if routing_client.get_fallback_handle(active_llm_config.handle):
+ await routing_client.record_success(active_llm_config.handle)
# If you've reached this point without an error, break out of retry loop
break
except ValueError as e:
@@ -1061,59 +1061,34 @@ class LettaAgentV3(LettaAgentV2):
self.stop_reason = LettaStopReason(stop_reason=StopReasonType.invalid_llm_response.value)
raise e
except (LLMRateLimitError, LLMServerError, LLMProviderOverloaded) as e:
- if is_auto_mode and is_primary:
- routing_client = await get_llm_routing_client()
- await routing_client.record_failure(primary_handle)
+ # Check if there's a fallback route for the current model
+ routing_client = await get_llm_routing_client()
+ current_handle = active_llm_config.handle
+ fallback_handle = routing_client.get_fallback_handle(current_handle)
- fallback_config = await routing_client.get_fallback_config(
+ if fallback_handle:
+ await routing_client.record_failure(current_handle)
+
+ fallback_config = await routing_client.get_fallback_config_for_handle(
+ fallback_handle=fallback_handle,
stored_llm_config=self.agent_state.llm_config,
actor=self.actor,
)
self.logger.warning(
- f"[AUTO MODE]: primary {primary_handle} failed ({type(e).__name__}), falling back to {fallback_config.handle}"
+ f"[LLM ROUTER]: {current_handle} failed ({type(e).__name__}), falling back to {fallback_config.handle}"
)
- fallback_client = LLMClient.create(
+
+ # Switch to fallback for this attempt and any subsequent retries (e.g. compaction)
+ active_llm_config = fallback_config
+ active_llm_client = LLMClient.create(
provider_type=fallback_config.model_endpoint_type,
put_inner_thoughts_first=True,
actor=self.actor,
)
- fallback_request_data = fallback_client.build_request_data(
- agent_type=self.agent_state.agent_type,
- messages=messages,
- llm_config=fallback_config,
- tools=valid_tools,
- force_tool_call=force_tool_call,
- requires_subsequent_tool_call=self._require_tool_call,
- tool_return_truncation_chars=self._compute_tool_return_truncation_chars(),
- )
- fallback_adapter = SimpleLLMRequestAdapter(
- llm_client=fallback_client,
- llm_config=fallback_config,
- call_type=LLMCallType.agent_step,
- agent_id=self.agent_state.id,
- agent_tags=self.agent_state.tags,
- run_id=run_id,
- org_id=self.actor.organization_id,
- user_id=self.actor.id,
- )
- fallback_invocation = fallback_adapter.invoke_llm(
- request_data=fallback_request_data,
- messages=messages,
- tools=valid_tools,
- use_assistant_message=False,
- requires_approval_tools=self.tool_rules_solver.get_requires_approval_tools(
- set([t["name"] for t in valid_tools])
- )
- + [ct.name for ct in self.client_tools],
- step_id=step_id,
- actor=self.actor,
- )
- async for chunk in fallback_invocation:
- if fallback_adapter.supports_token_streaming():
- if include_return_message_types is None or chunk.message_type in include_return_message_types:
- yield chunk
- llm_adapter = fallback_adapter
- break
+ llm_adapter.llm_client = active_llm_client
+ llm_adapter.llm_config = active_llm_config
+ is_primary = False
+ continue
self.stop_reason = LettaStopReason(stop_reason=StopReasonType.llm_api_error.value)
raise e
except LLMError as e:
diff --git a/letta/services/llm_router/__init__.py b/letta/services/llm_router/__init__.py
index 813f6f93e..521cf0ea2 100644
--- a/letta/services/llm_router/__init__.py
+++ b/letta/services/llm_router/__init__.py
@@ -1,12 +1,12 @@
-"""LLM routing client with circuit breaker and auto mode support.
+"""LLM routing client with circuit breaker and fallback support.
Conditionally imports Redis-backed implementation or falls back to noop base.
"""
try:
- from .client import LLMRoutingClient, get_llm_routing_client
+ from .llm_router_client import LLMRoutingClient, get_llm_routing_client
except ImportError:
- from .client_base import LLMRoutingClient
+ from .llm_router_client_base import LLMRoutingClient
async def get_llm_routing_client():
return LLMRoutingClient()
diff --git a/letta/services/llm_router/client.py b/letta/services/llm_router/client.py
deleted file mode 100644
index 392b3fccd..000000000
--- a/letta/services/llm_router/client.py
+++ /dev/null
@@ -1,247 +0,0 @@
-"""Redis-backed LLM routing client for model health tracking and auto mode resolution.
-
-Tracks overload errors per model handle using Redis sorted sets for
-sliding window error counting. Works across multiple pods.
-
-The noop base class lives in client_base.py.
-"""
-
-import time
-from enum import Enum
-from typing import TYPE_CHECKING, Optional
-
-from letta.data_sources.redis_client import AsyncRedisClient
-from letta.log import get_logger
-
-from .client_base import LLMRoutingClient as LLMRoutingClientBase
-
-if TYPE_CHECKING:
- from letta.schemas.llm_config import LLMConfig
- from letta.schemas.user import User
-
-logger = get_logger(__name__)
-
-# Auto mode model routing
-AUTO_MODE_PRIMARY = "baseten/glm-5"
-AUTO_MODE_FALLBACK = "zai/glm-5"
-
-# Circuit breaker configuration
-ERROR_WINDOW_SECONDS = 60 # Sliding window for error counting
-ERROR_THRESHOLD = 1 # Errors needed to trip circuit
-OPEN_DURATION_SECONDS = 30 # How long circuit stays open
-
-# Redis key prefixes
-CIRCUIT_ERRORS_KEY = "circuit:errors:"
-CIRCUIT_STATE_KEY = "circuit:state:"
-CIRCUIT_OPENED_AT_KEY = "circuit:opened:"
-
-
-def _build_baseten_config() -> "LLMConfig":
- """Build a hardcoded LLMConfig for Baseten GLM-5 (not discoverable via provider sync)."""
- from letta.schemas.enums import ProviderCategory
- from letta.schemas.llm_config import LLMConfig
-
- return LLMConfig(
- model="zai-org/GLM-5",
- model_endpoint_type="baseten",
- model_endpoint="3m5zok9w",
- context_window=180000,
- max_tokens=16384,
- handle="baseten/glm-5",
- provider_name="baseten",
- provider_category=ProviderCategory.base,
- )
-
-
-def _build_fireworks_config() -> "LLMConfig":
- """Build a hardcoded LLMConfig for Fireworks GLM-5 (not discoverable via provider sync)."""
- from letta.schemas.enums import ProviderCategory
- from letta.schemas.llm_config import LLMConfig
- from letta.settings import model_settings
-
- return LLMConfig(
- model="accounts/fireworks/models/glm-5",
- model_endpoint_type="fireworks",
- model_endpoint=model_settings.fireworks_api_base,
- context_window=180000,
- max_tokens=16384,
- handle="fireworks/glm-5",
- provider_name="fireworks",
- provider_category=ProviderCategory.base,
- )
-
-
-class CircuitState(str, Enum):
- CLOSED = "CLOSED"
- OPEN = "OPEN"
-
-
-class LLMRoutingClient(LLMRoutingClientBase):
- """Redis-backed LLM routing client for model health tracking and auto mode routing.
-
- Uses an internal circuit breaker (Redis sorted sets) for sliding window error
- counting across pods. The circuit breaker is an implementation detail.
- """
-
- def __init__(self, redis_client: AsyncRedisClient):
- self.redis = redis_client
-
- async def resolve_auto_mode_config(
- self,
- stored_llm_config: "LLMConfig",
- actor: "User",
- ) -> tuple["LLMConfig", bool, str]:
- """Resolve an auto mode handle to an actual model config.
-
- Uses the internal circuit breaker to decide between primary and fallback.
- Preserves user-configured context_window and max_tokens from the stored config.
-
- Args:
- stored_llm_config: The agent's stored LLM config (with auto mode handle).
- actor: The user actor for provider lookups.
-
- Returns:
- Tuple of (resolved_config, is_primary, primary_handle).
- """
- from letta.services.provider_manager import ProviderManager
- from letta.settings import model_settings
-
- provider_manager = ProviderManager()
- primary_handle = AUTO_MODE_PRIMARY
-
- if not model_settings.auto_mode_enabled:
- logger.info(f"[AUTO MODE]: primary {primary_handle} disabled via kill switch, using {AUTO_MODE_FALLBACK}")
- elif await self._is_healthy(primary_handle):
- config = _build_baseten_config()
- config = config.model_copy(
- update={
- "context_window": min(stored_llm_config.context_window, config.context_window),
- "max_tokens": min(stored_llm_config.max_tokens, config.max_tokens),
- }
- )
- return config, True, primary_handle
- else:
- logger.warning(f"[AUTO MODE]: primary {primary_handle} unhealthy, using {AUTO_MODE_FALLBACK}")
-
- config = await provider_manager.get_llm_config_from_handle(AUTO_MODE_FALLBACK, actor)
- config = config.model_copy(
- update={
- "context_window": min(stored_llm_config.context_window, config.context_window),
- "max_tokens": min(stored_llm_config.max_tokens, config.max_tokens),
- }
- )
- return config, False, primary_handle
-
- async def record_failure(self, handle: str) -> None:
- """Record a failure for a model handle. May trip the circuit.
-
- Args:
- handle: The model handle that failed.
- """
- client = await self.redis.get_client()
- now = time.time()
- errors_key = f"{CIRCUIT_ERRORS_KEY}{handle}"
-
- await client.zadd(errors_key, {str(now): now})
- await client.zremrangebyscore(errors_key, 0, now - ERROR_WINDOW_SECONDS)
- await client.expire(errors_key, ERROR_WINDOW_SECONDS * 2)
-
- error_count = await client.zcard(errors_key)
- if error_count >= ERROR_THRESHOLD:
- await self._open_circuit(client, handle)
-
- async def record_success(self, handle: str) -> None:
- """Record a success for a model handle. Closes circuit if open.
-
- Args:
- handle: The model handle that succeeded.
- """
- state = await self._get_state(handle)
- if state == CircuitState.OPEN:
- await self._close_circuit(handle)
-
- async def get_fallback_config(
- self,
- stored_llm_config: "LLMConfig",
- actor: "User",
- ) -> "LLMConfig":
- """Get the fallback config for auto mode.
-
- Args:
- stored_llm_config: The agent's stored LLM config (with auto mode handle).
- actor: The user actor for provider lookups.
-
- Returns:
- The fallback LLM config with preserved context_window and max_tokens.
- """
- from letta.services.provider_manager import ProviderManager
-
- provider_manager = ProviderManager()
- fallback_config = await provider_manager.get_llm_config_from_handle(AUTO_MODE_FALLBACK, actor)
- fallback_config = fallback_config.model_copy(
- update={
- "context_window": min(stored_llm_config.context_window, fallback_config.context_window),
- "max_tokens": min(stored_llm_config.max_tokens, fallback_config.max_tokens),
- }
- )
- return fallback_config
-
- async def _is_healthy(self, handle: str) -> bool:
- """Check if a model handle is healthy (internal circuit breaker).
-
- Returns True if the circuit is closed, or if the open duration has elapsed
- (allowing a retry probe).
-
- Args:
- handle: The model handle to check.
-
- Returns:
- True if the model should be tried, False if it should be skipped.
- """
- state = await self._get_state(handle)
- if state == CircuitState.CLOSED:
- return True
-
- opened_at = await self._get_opened_at(handle)
- if opened_at and time.time() - opened_at >= OPEN_DURATION_SECONDS:
- return True
-
- return False
-
- async def _get_state(self, handle: str) -> CircuitState:
- client = await self.redis.get_client()
- state = await client.get(f"{CIRCUIT_STATE_KEY}{handle}")
- return CircuitState.OPEN if state == CircuitState.OPEN.value else CircuitState.CLOSED
-
- async def _get_opened_at(self, handle: str) -> Optional[float]:
- client = await self.redis.get_client()
- val = await client.get(f"{CIRCUIT_OPENED_AT_KEY}{handle}")
- return float(val) if val else None
-
- async def _open_circuit(self, client, handle: str) -> None:
- now = time.time()
- await client.set(f"{CIRCUIT_STATE_KEY}{handle}", CircuitState.OPEN.value, ex=OPEN_DURATION_SECONDS * 2)
- await client.set(f"{CIRCUIT_OPENED_AT_KEY}{handle}", str(now), ex=OPEN_DURATION_SECONDS * 2)
- logger.warning(f"Circuit OPENED for model {handle}")
-
- async def _close_circuit(self, handle: str) -> None:
- client = await self.redis.get_client()
- await client.delete(f"{CIRCUIT_STATE_KEY}{handle}", f"{CIRCUIT_OPENED_AT_KEY}{handle}", f"{CIRCUIT_ERRORS_KEY}{handle}")
- logger.info(f"Circuit CLOSED for model {handle}")
-
-
-async def get_llm_routing_client() -> LLMRoutingClientBase:
- """Factory that returns the appropriate LLM routing client instance.
-
- Returns the Redis-backed implementation if Redis is available,
- otherwise returns the noop base.
-
- Returns:
- An LLMRoutingClient instance.
- """
- from letta.data_sources.redis_client import NoopAsyncRedisClient, get_redis_client
-
- redis_client = await get_redis_client()
- if isinstance(redis_client, NoopAsyncRedisClient):
- return LLMRoutingClientBase()
- return LLMRoutingClient(redis_client)
diff --git a/letta/services/llm_router/client_base.py b/letta/services/llm_router/llm_router_client_base.py
similarity index 65%
rename from letta/services/llm_router/client_base.py
rename to letta/services/llm_router/llm_router_client_base.py
index 3bb9f8159..bce6f69a9 100644
--- a/letta/services/llm_router/client_base.py
+++ b/letta/services/llm_router/llm_router_client_base.py
@@ -1,10 +1,10 @@
"""Base LLM routing client.
This is the base (OSS/self-hosted) implementation without Redis support.
-The Redis-backed implementation lives in client.py.
+The Redis-backed implementation lives in llm_router_client.py.
"""
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Optional
if TYPE_CHECKING:
from letta.schemas.llm_config import LLMConfig
@@ -56,12 +56,38 @@ class LLMRoutingClient:
stored_llm_config: The agent's stored LLM config.
actor: The user actor for provider lookups.
- Returns:
- The fallback LLM config.
-
Raises:
RuntimeError: Auto mode requires Redis for circuit breaker support.
"""
raise RuntimeError(
"Auto mode requires Redis for circuit breaker support. Configure Redis or disable auto_mode_enabled in settings."
)
+
+ async def get_fallback_config_for_handle(
+ self,
+ fallback_handle: str,
+ stored_llm_config: "LLMConfig",
+ actor: "User",
+ ) -> "LLMConfig":
+ """Get a fallback config for any handle (noop in base).
+
+ Args:
+ fallback_handle: The fallback model handle to resolve.
+ stored_llm_config: The agent's stored LLM config.
+ actor: The user actor for provider lookups.
+
+ Raises:
+ RuntimeError: Fallback routing requires Redis for circuit breaker support.
+ """
+ raise RuntimeError("Fallback routing requires Redis for circuit breaker support.")
+
+ def get_fallback_handle(self, handle: str) -> Optional[str]:
+ """Get the fallback handle for a given primary handle (noop in base).
+
+ Args:
+ handle: The primary model handle.
+
+ Returns:
+ None — no fallback routes configured without Redis.
+ """
+ return None
diff --git a/letta/services/summarizer/compact.py b/letta/services/summarizer/compact.py
index 68c721b8f..8c4d0ee61 100644
--- a/letta/services/summarizer/compact.py
+++ b/letta/services/summarizer/compact.py
@@ -49,6 +49,9 @@ async def build_summarizer_llm_config(
then apply any explicit ``compaction_settings.model_settings`` via
``_to_legacy_config_params``.
+ For auto mode agents (letta/baseten providers), uses the LLM router's
+ circuit breaker to try the primary model first, falling back if unhealthy.
+
Args:
agent_llm_config: The agent's LLM configuration to use as base.
summarizer_config: Compaction settings with optional model override.
@@ -59,6 +62,20 @@ async def build_summarizer_llm_config(
"""
from letta.schemas.enums import ProviderType
+ # Auto mode agents: use the LLM router to resolve primary vs fallback
+ if not summarizer_config.model and agent_llm_config.provider_name in ("letta", "baseten"):
+ try:
+ from letta.services.llm_router import get_llm_routing_client
+
+ routing_client = await get_llm_routing_client()
+ resolved_config, _, _ = await routing_client.resolve_auto_mode_config(
+ stored_llm_config=agent_llm_config,
+ actor=actor,
+ )
+ return resolved_config
+ except Exception as e:
+ logger.warning(f"Failed to resolve auto mode config for summarizer: {e}. Falling back to defaults.")
+
# If no summarizer model specified, use lightweight provider-specific defaults
if not summarizer_config.model:
provider_name = agent_llm_config.provider_name or agent_llm_config.model_endpoint_type
diff --git a/letta/services/summarizer/summarizer_config.py b/letta/services/summarizer/summarizer_config.py
index a1cb46a66..bbfe7f631 100644
--- a/letta/services/summarizer/summarizer_config.py
+++ b/letta/services/summarizer/summarizer_config.py
@@ -14,6 +14,7 @@ def get_default_summarizer_model(provider_type: ProviderType) -> str | None:
ProviderType.anthropic: "anthropic/claude-haiku-4-5",
ProviderType.openai: "openai/gpt-5-mini",
ProviderType.google_ai: "google_ai/gemini-2.5-flash",
+ ProviderType.letta: "zai/glm-5",
}
return summarizer_defaults.get(provider_type)