Platform for stateful agents: AI with advanced memory that can learn and self-improve over time.
feat(core): Add LLM routing with auto mode and circuit breaker (#9742)
cthomas343ef9096133
diff --git a/letta/agents/base_agent.py b/letta/agents/base_agent.pyindex 3e6019a4b..c81f760d4 100644--- a/letta/agents/base_agent.py+++ b/letta/agents/base_agent.py@@ -56,6 +56,7 @@ class BaseAgent(ABC):input_messages: List[MessageCreate],max_steps: int = DEFAULT_MAX_STEPS,run_id: Optional[str] = None,+ auto_mode_enabled: bool = False,billing_context: "BillingContext | None" = None,) -> LettaResponse:"""diff --git a/letta/agents/base_agent_v2.py b/letta/agents/base_agent_v2.pyindex 515edb76b..0edbee0ce 100644--- a/letta/agents/base_agent_v2.py+++ b/letta/agents/base_agent_v2.py@@ -53,6 +53,7 @@ class BaseAgentV2(ABC):request_start_timestamp_ns: int | None = None,client_tools: list["ClientToolSchema"] | None = None,include_compaction_messages: bool = False, # Not used in V2, but accepted for API compatibility+ auto_mode_enabled: bool = False, # Not used in V2, but accepted for API compatibilitybilling_context: "BillingContext | None" = None,) -> LettaResponse:"""@@ -78,6 +79,7 @@ class BaseAgentV2(ABC):conversation_id: str | None = None,client_tools: list["ClientToolSchema"] | None = None,include_compaction_messages: bool = False, # Not used in V2, but accepted for API compatibility+ auto_mode_enabled: bool = False, # Not used in V2, but accepted for API compatibilitybilling_context: "BillingContext | None" = None,) -> AsyncGenerator[LettaMessage | LegacyLettaMessage | MessageStreamStatus, None]:"""diff --git a/letta/agents/letta_agent_v2.py b/letta/agents/letta_agent_v2.pyindex 13fb9b07f..b958c9fde 100644--- a/letta/agents/letta_agent_v2.py+++ b/letta/agents/letta_agent_v2.py@@ -186,6 +186,7 @@ class LettaAgentV2(BaseAgentV2):request_start_timestamp_ns: int | None = None,client_tools: list[ClientToolSchema] | None = None,include_compaction_messages: bool = False, # Not used in V2, but accepted for API compatibility+ auto_mode_enabled: bool = False, # Not used in V2, but accepted for API compatibilitybilling_context: "BillingContext | None" = None,) -> LettaResponse:"""@@ -292,6 +293,7 @@ class LettaAgentV2(BaseAgentV2):conversation_id: str | None = None, # Not used in V2, but accepted for API compatibilityclient_tools: list[ClientToolSchema] | None = None,include_compaction_messages: bool = False, # Not used in V2, but accepted for API compatibility+ auto_mode_enabled: bool = False, # Not used in V2, but accepted for API compatibilitybilling_context: BillingContext | None = None,) -> AsyncGenerator[str, None]:"""diff --git a/letta/agents/letta_agent_v3.py b/letta/agents/letta_agent_v3.pyindex 3a77c011e..baa159275 100644--- a/letta/agents/letta_agent_v3.py+++ b/letta/agents/letta_agent_v3.py@@ -21,10 +21,19 @@ from letta.agents.helpers import ()from letta.agents.letta_agent_v2 import LettaAgentV2from letta.constants import DEFAULT_MAX_STEPS, NON_USER_MSG_PREFIX, REQUEST_HEARTBEAT_PARAM-from letta.errors import ContextWindowExceededError, LLMEmptyResponseError, LLMError, SystemPromptTokenExceededError+from letta.errors import (+ ContextWindowExceededError,+ LLMEmptyResponseError,+ LLMError,+ LLMProviderOverloaded,+ LLMRateLimitError,+ LLMServerError,+ SystemPromptTokenExceededError,+)from letta.helpers import ToolRulesSolverfrom letta.helpers.datetime_helpers import get_utc_time, get_utc_timestamp_nsfrom letta.helpers.tool_execution_helper import enable_strict_mode+from letta.llm_api.llm_client import LLMClientfrom letta.local_llm.constants import INNER_THOUGHTS_KWARGfrom letta.otel.tracing import trace_methodfrom letta.schemas.agent import AgentState@@ -58,6 +67,8 @@ from letta.server.rest_api.utils import ()from letta.services.conversation_manager import ConversationManagerfrom letta.services.helpers.tool_parser_helper import runtime_override_tool_json_schema+from letta.services.llm_router import get_llm_routing_client+from letta.services.provider_manager import AUTO_MODE_HANDLESfrom letta.services.summarizer.compact import compact_messagesfrom letta.services.summarizer.summarizer_config import CompactionSettingsfrom letta.services.summarizer.summarizer_sliding_window import count_tokens@@ -150,6 +161,7 @@ class LettaAgentV3(LettaAgentV2):conversation_id: str | None = None,client_tools: list[ClientToolSchema] | None = None,include_compaction_messages: bool = False,+ auto_mode_enabled: bool = False,billing_context: "BillingContext | None" = None,) -> LettaResponse:"""@@ -261,6 +273,7 @@ class LettaAgentV3(LettaAgentV2):include_return_message_types=include_return_message_types,request_start_timestamp_ns=request_start_timestamp_ns,include_compaction_messages=include_compaction_messages,+ auto_mode_enabled=auto_mode_enabled,)input_messages_to_persist = [] # clear after first step@@ -365,6 +378,7 @@ class LettaAgentV3(LettaAgentV2):conversation_id: str | None = None,client_tools: list[ClientToolSchema] | None = None,include_compaction_messages: bool = False,+ auto_mode_enabled: bool = False,billing_context: BillingContext | None = None,) -> AsyncGenerator[str, None]:"""@@ -493,6 +507,7 @@ class LettaAgentV3(LettaAgentV2):include_return_message_types=include_return_message_types,request_start_timestamp_ns=request_start_timestamp_ns,include_compaction_messages=include_compaction_messages,+ auto_mode_enabled=auto_mode_enabled,)input_messages_to_persist = [] # clear after first stepasync for chunk in response:@@ -792,6 +807,7 @@ class LettaAgentV3(LettaAgentV2):dry_run: bool = False,enforce_run_id_set: bool = True,include_compaction_messages: bool = False,+ auto_mode_enabled: bool = False,) -> AsyncGenerator[LettaMessage | dict, None]:"""Execute a single agent step (one LLM call and tool execution).@@ -923,13 +939,37 @@ class LettaAgentV3(LettaAgentV2):step_id=step_id, run_id=run_id)+ # Auto mode: resolve handle to actual model config+ auto_mode_handle = self.agent_state.llm_config.handle+ is_auto_mode = auto_mode_handle in AUTO_MODE_HANDLES+ is_primary = False+ primary_handle = ""++ if is_auto_mode:+ routing_client = await get_llm_routing_client()+ active_llm_config, is_primary, primary_handle = await routing_client.resolve_auto_mode_config(+ auto_mode_enabled=auto_mode_enabled,+ stored_llm_config=self.agent_state.llm_config,+ actor=self.actor,+ )+ if not is_primary:+ self.logger.info(f"[AUTO MODE]: primary {primary_handle} rerouted, falling back to {active_llm_config.handle}")+ active_llm_client = LLMClient.create(+ provider_type=active_llm_config.model_endpoint_type,+ put_inner_thoughts_first=True,+ actor=self.actor,+ )+ else:+ active_llm_config = self.agent_state.llm_config+ active_llm_client = self.llm_client+force_tool_call = valid_tools[0]["name"] if len(valid_tools) == 1 and self._require_tool_call else Nonefor llm_request_attempt in range(summarizer_settings.max_summarizer_retries + 1):try:- request_data = self.llm_client.build_request_data(+ request_data = active_llm_client.build_request_data(agent_type=self.agent_state.agent_type,messages=messages,- llm_config=self.agent_state.llm_config,+ llm_config=active_llm_config,tools=valid_tools,force_tool_call=force_tool_call,requires_subsequent_tool_call=self._require_tool_call,@@ -945,30 +985,30 @@ class LettaAgentV3(LettaAgentV2):)# Anthropic/Bedrock/MiniMax parallel tool use (MiniMax uses Anthropic-compatible API)- if self.agent_state.llm_config.model_endpoint_type in ["anthropic", "bedrock", "minimax"]:+ if active_llm_config.model_endpoint_type in ["anthropic", "bedrock", "minimax"]:if (isinstance(request_data.get("tool_choice"), dict)and "disable_parallel_tool_use" in request_data["tool_choice"]):# Gate parallel tool use on both: no tool rules and toggled on- if no_tool_rules and self.agent_state.llm_config.parallel_tool_calls:+ if no_tool_rules and active_llm_config.parallel_tool_calls:request_data["tool_choice"]["disable_parallel_tool_use"] = Falseelse:# Explicitly disable when tool rules present or llm_config toggled offrequest_data["tool_choice"]["disable_parallel_tool_use"] = True# OpenAI parallel tool use- elif self.agent_state.llm_config.model_endpoint_type == "openai":+ elif active_llm_config.model_endpoint_type == "openai":# For OpenAI, we control parallel tool calling via parallel_tool_calls field# Only allow parallel tool calls when no tool rules and enabled in configif "parallel_tool_calls" in request_data:- if no_tool_rules and self.agent_state.llm_config.parallel_tool_calls:+ if no_tool_rules and active_llm_config.parallel_tool_calls:request_data["parallel_tool_calls"] = Trueelse:request_data["parallel_tool_calls"] = False# Gemini (Google AI/Vertex) parallel tool use- elif self.agent_state.llm_config.model_endpoint_type in ["google_ai", "google_vertex"]:+ elif active_llm_config.model_endpoint_type in ["google_ai", "google_vertex"]:# Gemini supports parallel tool calling natively through multiple parts in the response# We just need to ensure the config flag is set for tracking purposes# The actual handling happens in GoogleVertexClient.convert_response_to_chat_completion@@ -997,6 +1037,10 @@ class LettaAgentV3(LettaAgentV2):if llm_adapter.supports_token_streaming():if include_return_message_types is None or chunk.message_type in include_return_message_types:yield chunk+ # Report success to circuit breaker+ if is_auto_mode and is_primary:+ routing_client = await get_llm_routing_client()+ await routing_client.record_success(primary_handle)# If you've reached this point without an error, break out of retry loopbreakexcept ValueError as e:@@ -1005,6 +1049,62 @@ class LettaAgentV3(LettaAgentV2):except LLMEmptyResponseError as e:self.stop_reason = LettaStopReason(stop_reason=StopReasonType.invalid_llm_response.value)raise e+ except (LLMRateLimitError, LLMServerError, LLMProviderOverloaded) as e:+ if is_auto_mode and is_primary:+ routing_client = await get_llm_routing_client()+ await routing_client.record_failure(primary_handle)++ fallback_config = await routing_client.get_fallback_config(+ stored_llm_config=self.agent_state.llm_config,+ actor=self.actor,+ )+ self.logger.warning(+ f"[AUTO MODE]: primary {primary_handle} failed ({type(e).__name__}), falling back to {fallback_config.handle}"+ )+ fallback_client = LLMClient.create(+ provider_type=fallback_config.model_endpoint_type,+ put_inner_thoughts_first=True,+ actor=self.actor,+ )+ fallback_request_data = fallback_client.build_request_data(+ agent_type=self.agent_state.agent_type,+ messages=messages,+ llm_config=fallback_config,+ tools=valid_tools,+ force_tool_call=force_tool_call,+ requires_subsequent_tool_call=self._require_tool_call,+ tool_return_truncation_chars=self._compute_tool_return_truncation_chars(),+ )+ fallback_adapter = SimpleLLMRequestAdapter(+ llm_client=fallback_client,+ llm_config=fallback_config,+ call_type=LLMCallType.agent_step,+ agent_id=self.agent_state.id,+ agent_tags=self.agent_state.tags,+ run_id=run_id,+ org_id=self.actor.organization_id,+ user_id=self.actor.id,+ )+ fallback_invocation = fallback_adapter.invoke_llm(+ request_data=fallback_request_data,+ messages=messages,+ tools=valid_tools,+ use_assistant_message=False,+ requires_approval_tools=self.tool_rules_solver.get_requires_approval_tools(+ set([t["name"] for t in valid_tools])+ )+ + [ct.name for ct in self.client_tools],+ step_id=step_id,+ actor=self.actor,+ )+ async for chunk in fallback_invocation:+ if fallback_adapter.supports_token_streaming():+ if include_return_message_types is None or chunk.message_type in include_return_message_types:+ yield chunk+ llm_adapter = fallback_adapter+ break+ self.stop_reason = LettaStopReason(stop_reason=StopReasonType.llm_api_error.value)+ raise eexcept LLMError as e:self.stop_reason = LettaStopReason(stop_reason=StopReasonType.llm_api_error.value)raise ediff --git a/letta/groups/sleeptime_multi_agent_v2.py b/letta/groups/sleeptime_multi_agent_v2.pyindex e842e0b7f..fb833b8d8 100644--- a/letta/groups/sleeptime_multi_agent_v2.py+++ b/letta/groups/sleeptime_multi_agent_v2.py@@ -70,6 +70,7 @@ class SleeptimeMultiAgentV2(BaseAgent):use_assistant_message: bool = True,request_start_timestamp_ns: int | None = None,include_return_message_types: list[MessageType] | None = None,+ auto_mode_enabled: bool = False, # Not used in multi-agent, but accepted for API compatibilitybilling_context: "BillingContext | None" = None,) -> LettaResponse:run_ids = []diff --git a/letta/groups/sleeptime_multi_agent_v3.py b/letta/groups/sleeptime_multi_agent_v3.pyindex 7a148eada..0192e9b1d 100644--- a/letta/groups/sleeptime_multi_agent_v3.py+++ b/letta/groups/sleeptime_multi_agent_v3.py@@ -48,6 +48,7 @@ class SleeptimeMultiAgentV3(LettaAgentV2):request_start_timestamp_ns: int | None = None,client_tools: list[ClientToolSchema] | None = None,include_compaction_messages: bool = False,+ auto_mode_enabled: bool = False, # Not used in multi-agent, but accepted for API compatibilitybilling_context: "BillingContext | None" = None,) -> LettaResponse:self.run_ids = []diff --git a/letta/groups/sleeptime_multi_agent_v4.py b/letta/groups/sleeptime_multi_agent_v4.pyindex 150cd3f7f..782094eae 100644--- a/letta/groups/sleeptime_multi_agent_v4.py+++ b/letta/groups/sleeptime_multi_agent_v4.py@@ -48,6 +48,7 @@ class SleeptimeMultiAgentV4(LettaAgentV3):conversation_id: str | None = None,client_tools: list[ClientToolSchema] | None = None,include_compaction_messages: bool = False,+ auto_mode_enabled: bool = False, # Not used in multi-agent, but accepted for API compatibilitybilling_context: "BillingContext | None" = None,) -> LettaResponse:self.run_ids = []diff --git a/letta/server/rest_api/dependencies.py b/letta/server/rest_api/dependencies.pyindex 66ea43bb4..15772d754 100644--- a/letta/server/rest_api/dependencies.py+++ b/letta/server/rest_api/dependencies.py@@ -20,6 +20,7 @@ class ExperimentalParams(BaseModel):letta_v1_agent: Optional[bool] = Noneletta_v1_agent_message_async: Optional[bool] = Nonemodal_sandbox: Optional[bool] = None+ auto_mode: Optional[bool] = Noneclass HeaderParams(BaseModel):@@ -46,6 +47,7 @@ def get_headers(None, alias="X-Experimental-Letta-V1-Agent-Message-Async", include_in_schema=False),modal_sandbox: Optional[str] = Header(None, alias="X-Experimental-Modal-Sandbox", include_in_schema=False),+ auto_mode: Optional[str] = Header(None, alias="X-Experimental-Auto-Mode", include_in_schema=False),billing_plan_type: Optional[str] = Header(None, alias="X-Billing-Plan-Type", include_in_schema=False),billing_cost_source: Optional[str] = Header(None, alias="X-Billing-Cost-Source", include_in_schema=False),billing_customer_id: Optional[str] = Header(None, alias="X-Billing-Customer-Id", include_in_schema=False),@@ -69,6 +71,7 @@ def get_headers(letta_v1_agent=(letta_v1_agent == "true") if letta_v1_agent else None,letta_v1_agent_message_async=(letta_v1_agent_message_async == "true") if letta_v1_agent_message_async else None,modal_sandbox=(modal_sandbox == "true") if modal_sandbox else None,+ auto_mode=(auto_mode == "true") if auto_mode else None,),billing_context=BillingContext(plan_type=billing_plan_type,diff --git a/letta/server/rest_api/routers/v1/agents.py b/letta/server/rest_api/routers/v1/agents.pyindex dce472409..962728d1b 100644--- a/letta/server/rest_api/routers/v1/agents.py+++ b/letta/server/rest_api/routers/v1/agents.py@@ -1692,12 +1692,14 @@ async def send_message(actor = await server.user_manager.get_actor_or_default_async(actor_id=headers.actor_id)if request.streaming and is_1_0_sdk:+ auto_mode_enabled = bool(headers.experimental_params and headers.experimental_params.auto_mode)streaming_service = StreamingService(server)run, result = await streaming_service.create_agent_stream(agent_id=agent_id,actor=actor,request=request,run_type="send_message",+ auto_mode_enabled=auto_mode_enabled,billing_context=headers.billing_context,)return result@@ -1759,6 +1761,7 @@ async def send_message(})+ auto_mode_enabled = bool(headers.experimental_params and headers.experimental_params.auto_mode)agent_loop = AgentLoop.load(agent_state=agent, actor=actor)result = await agent_loop.step(request.messages,@@ -1769,6 +1772,7 @@ async def send_message(include_return_message_types=request.include_return_message_types,client_tools=request.client_tools,include_compaction_messages=request.include_compaction_messages,+ auto_mode_enabled=auto_mode_enabled,billing_context=headers.billing_context,)run_status = result.stop_reason.stop_reason.run_statusdiff --git a/letta/server/server.py b/letta/server/server.pyindex ea6eef31b..23e2bf62e 100644--- a/letta/server/server.py+++ b/letta/server/server.py@@ -1240,9 +1240,25 @@ class SyncServer(object):)# Build LLMConfig objects from database+ from letta.services.provider_manager import AUTO_MODE_HANDLES+provider_cache: Dict[str, Provider] = {}typed_provider_cache: Dict[str, Any] = {}for model in provider_models:+ # Handle synthetic auto mode models separately+ if model.handle in AUTO_MODE_HANDLES:+ llm_config = LLMConfig(+ model=model.name,+ model_endpoint_type=model.model_endpoint_type,+ model_endpoint="",+ context_window=model.max_context_window or 180000,+ handle=model.handle,+ provider_name="letta",+ max_tokens=8192,+ )+ llm_models.append(llm_config)+ continue+# Get provider details (with caching to avoid N+1 queries)if model.provider_id not in provider_cache:provider_cache[model.provider_id] = await self.provider_manager.get_provider_async(model.provider_id, actor)diff --git a/letta/services/llm_router/__init__.py b/letta/services/llm_router/__init__.pynew file mode 100644index 000000000..813f6f93e--- /dev/null+++ b/letta/services/llm_router/__init__.py@@ -0,0 +1,18 @@+"""LLM routing client with circuit breaker and auto mode support.++Conditionally imports Redis-backed implementation or falls back to noop base.+"""++try:+ from .client import LLMRoutingClient, get_llm_routing_client+except ImportError:+ from .client_base import LLMRoutingClient++ async def get_llm_routing_client():+ return LLMRoutingClient()+++__all__ = [+ "LLMRoutingClient",+ "get_llm_routing_client",+]diff --git a/letta/services/llm_router/client.py b/letta/services/llm_router/client.pynew file mode 100644index 000000000..e7659f1aa--- /dev/null+++ b/letta/services/llm_router/client.py@@ -0,0 +1,212 @@+"""Redis-backed LLM routing client for model health tracking and auto mode resolution.++Tracks overload errors per model handle using Redis sorted sets for+sliding window error counting. Works across multiple pods.++The noop base class lives in client_base.py.+"""++import time+from enum import Enum+from typing import TYPE_CHECKING, Optional++from letta.data_sources.redis_client import AsyncRedisClient+from letta.log import get_logger++from .client_base import LLMRoutingClient as LLMRoutingClientBase++if TYPE_CHECKING:+ from letta.schemas.llm_config import LLMConfig+ from letta.schemas.user import User++logger = get_logger(__name__)++# Auto mode model routing+AUTO_MODE_PRIMARY = "zai/glm-5"+AUTO_MODE_FALLBACK = "zai/glm-5"++# Circuit breaker configuration+ERROR_WINDOW_SECONDS = 60 # Sliding window for error counting+ERROR_THRESHOLD = 1 # Errors needed to trip circuit+OPEN_DURATION_SECONDS = 30 # How long circuit stays open++# Redis key prefixes+CIRCUIT_ERRORS_KEY = "circuit:errors:"+CIRCUIT_STATE_KEY = "circuit:state:"+CIRCUIT_OPENED_AT_KEY = "circuit:opened:"+++class CircuitState(str, Enum):+ CLOSED = "CLOSED"+ OPEN = "OPEN"+++class LLMRoutingClient(LLMRoutingClientBase):+ """Redis-backed LLM routing client for model health tracking and auto mode routing.++ Uses an internal circuit breaker (Redis sorted sets) for sliding window error+ counting across pods. The circuit breaker is an implementation detail.+ """++ def __init__(self, redis_client: AsyncRedisClient):+ self.redis = redis_client++ async def resolve_auto_mode_config(+ self,+ auto_mode_enabled: bool,+ stored_llm_config: "LLMConfig",+ actor: "User",+ ) -> tuple["LLMConfig", bool, str]:+ """Resolve an auto mode handle to an actual model config.++ Uses the internal circuit breaker to decide between primary and fallback.+ Preserves user-configured context_window and max_tokens from the stored config.++ Args:+ auto_mode_enabled: Whether the experimental auto_mode header was set.+ stored_llm_config: The agent's stored LLM config (with auto mode handle).+ actor: The user actor for provider lookups.++ Returns:+ Tuple of (resolved_config, is_primary, primary_handle).+ """+ from letta.services.provider_manager import ProviderManager++ provider_manager = ProviderManager()+ primary_handle = AUTO_MODE_PRIMARY++ if auto_mode_enabled:+ if await self._is_healthy(primary_handle):+ config = await provider_manager.get_llm_config_from_handle(primary_handle, actor)+ config = config.model_copy(+ update={+ "context_window": min(stored_llm_config.context_window, config.context_window),+ "max_tokens": min(stored_llm_config.max_tokens, config.max_tokens),+ }+ )+ return config, True, primary_handle++ logger.warning(f"Auto mode: primary {primary_handle} unhealthy, using {AUTO_MODE_FALLBACK}")++ config = await provider_manager.get_llm_config_from_handle(AUTO_MODE_FALLBACK, actor)+ config = config.model_copy(+ update={+ "context_window": min(stored_llm_config.context_window, config.context_window),+ "max_tokens": min(stored_llm_config.max_tokens, config.max_tokens),+ }+ )+ return config, False, primary_handle++ async def record_failure(self, handle: str) -> None:+ """Record a failure for a model handle. May trip the circuit.++ Args:+ handle: The model handle that failed.+ """+ client = await self.redis.get_client()+ now = time.time()+ errors_key = f"{CIRCUIT_ERRORS_KEY}{handle}"++ await client.zadd(errors_key, {str(now): now})+ await client.zremrangebyscore(errors_key, 0, now - ERROR_WINDOW_SECONDS)+ await client.expire(errors_key, ERROR_WINDOW_SECONDS * 2)++ error_count = await client.zcard(errors_key)+ if error_count >= ERROR_THRESHOLD:+ await self._open_circuit(client, handle)++ async def record_success(self, handle: str) -> None:+ """Record a success for a model handle. Closes circuit if open.++ Args:+ handle: The model handle that succeeded.+ """+ state = await self._get_state(handle)+ if state == CircuitState.OPEN:+ await self._close_circuit(handle)++ async def get_fallback_config(+ self,+ stored_llm_config: "LLMConfig",+ actor: "User",+ ) -> "LLMConfig":+ """Get the fallback config for auto mode.++ Args:+ stored_llm_config: The agent's stored LLM config (with auto mode handle).+ actor: The user actor for provider lookups.++ Returns:+ The fallback LLM config with preserved context_window and max_tokens.+ """+ from letta.services.provider_manager import ProviderManager++ provider_manager = ProviderManager()+ fallback_config = await provider_manager.get_llm_config_from_handle(AUTO_MODE_FALLBACK, actor)+ fallback_config = fallback_config.model_copy(+ update={+ "context_window": min(stored_llm_config.context_window, fallback_config.context_window),+ "max_tokens": min(stored_llm_config.max_tokens, fallback_config.max_tokens),+ }+ )+ return fallback_config++ async def _is_healthy(self, handle: str) -> bool:+ """Check if a model handle is healthy (internal circuit breaker).++ Returns True if the circuit is closed, or if the open duration has elapsed+ (allowing a retry probe).++ Args:+ handle: The model handle to check.++ Returns:+ True if the model should be tried, False if it should be skipped.+ """+ state = await self._get_state(handle)+ if state == CircuitState.CLOSED:+ return True++ opened_at = await self._get_opened_at(handle)+ if opened_at and time.time() - opened_at >= OPEN_DURATION_SECONDS:+ return True++ return False++ async def _get_state(self, handle: str) -> CircuitState:+ client = await self.redis.get_client()+ state = await client.get(f"{CIRCUIT_STATE_KEY}{handle}")+ return CircuitState.OPEN if state == CircuitState.OPEN.value else CircuitState.CLOSED++ async def _get_opened_at(self, handle: str) -> Optional[float]:+ client = await self.redis.get_client()+ val = await client.get(f"{CIRCUIT_OPENED_AT_KEY}{handle}")+ return float(val) if val else None++ async def _open_circuit(self, client, handle: str) -> None:+ now = time.time()+ await client.set(f"{CIRCUIT_STATE_KEY}{handle}", CircuitState.OPEN.value, ex=OPEN_DURATION_SECONDS * 2)+ await client.set(f"{CIRCUIT_OPENED_AT_KEY}{handle}", str(now), ex=OPEN_DURATION_SECONDS * 2)+ logger.warning(f"Circuit OPENED for model {handle}")++ async def _close_circuit(self, handle: str) -> None:+ client = await self.redis.get_client()+ await client.delete(f"{CIRCUIT_STATE_KEY}{handle}", f"{CIRCUIT_OPENED_AT_KEY}{handle}", f"{CIRCUIT_ERRORS_KEY}{handle}")+ logger.info(f"Circuit CLOSED for model {handle}")+++async def get_llm_routing_client() -> LLMRoutingClientBase:+ """Factory that returns the appropriate LLM routing client instance.++ Returns the Redis-backed implementation if Redis is available,+ otherwise returns the noop base.++ Returns:+ An LLMRoutingClient instance.+ """+ from letta.data_sources.redis_client import NoopAsyncRedisClient, get_redis_client++ redis_client = await get_redis_client()+ if isinstance(redis_client, NoopAsyncRedisClient):+ return LLMRoutingClientBase()+ return LLMRoutingClient(redis_client)diff --git a/letta/services/llm_router/client_base.py b/letta/services/llm_router/client_base.pynew file mode 100644index 000000000..3c69c8d01--- /dev/null+++ b/letta/services/llm_router/client_base.py@@ -0,0 +1,69 @@+"""Base LLM routing client.++This is the base (OSS/self-hosted) implementation without Redis support.+The Redis-backed implementation lives in client.py.+"""++from typing import TYPE_CHECKING++if TYPE_CHECKING:+ from letta.schemas.llm_config import LLMConfig+ from letta.schemas.user import User+++class LLMRoutingClient:+ """Base LLM routing client.++ Used when Redis is not configured (OSS/self-hosted).+ Auto mode is not supported without Redis.+ """++ async def resolve_auto_mode_config(+ self,+ auto_mode_enabled: bool,+ stored_llm_config: "LLMConfig",+ actor: "User",+ ) -> tuple["LLMConfig", bool, str]:+ """Resolve an auto mode handle to an actual model config.++ Args:+ auto_mode_enabled: Whether the experimental auto_mode header was set.+ stored_llm_config: The agent's stored LLM config (with auto mode handle).+ actor: The user actor for provider lookups.++ Returns:+ Tuple of (resolved_config, is_primary, primary_handle).++ Raises:+ RuntimeError: Auto mode requires Redis for circuit breaker support.+ """+ raise RuntimeError(+ "Auto mode requires Redis for circuit breaker support. Configure Redis or disable auto_mode_enabled in settings."+ )++ async def record_failure(self, handle: str) -> None:+ """Record a failure for a model handle (noop in base)."""++ async def record_success(self, handle: str) -> None:+ """Record a success for a model handle (noop in base)."""++ async def get_fallback_config(+ self,+ stored_llm_config: "LLMConfig",+ actor: "User",+ ) -> "LLMConfig":+ """Get the fallback config for auto mode.++ Args:+ stored_llm_config: The agent's stored LLM config.+ actor: The user actor for provider lookups.++ Returns:+ The fallback LLM config.++ Raises:+ RuntimeError: Auto mode requires Redis for circuit breaker support.+ """+ raise RuntimeError(+ "Auto mode requires Redis for circuit breaker support. Configure Redis or disable auto_mode_enabled in settings."+ )diff --git a/letta/services/provider_manager.py b/letta/services/provider_manager.pyindex d1c98dc52..2c8359488 100644--- a/letta/services/provider_manager.py+++ b/letta/services/provider_manager.py@@ -1,3 +1,4 @@+import hashlibfrom typing import List, Optional, Tuple, Unionfrom sqlalchemy import and_, select@@ -19,6 +20,9 @@ from letta.validators import raise_on_invalid_idlogger = get_logger(__name__)+# Auto mode model handles+AUTO_MODE_HANDLES = ["letta/auto", "letta/auto-fast"]+class ProviderManager:@enforce_types@@ -929,7 +933,31 @@ class ProviderManager:all_models = {(m.handle, m.model_type): m for m in global_models}all_models.update({(m.handle, m.model_type): m for m in org_models})- return [m.to_pydantic() for m in all_models.values()]+ models = [m.to_pydantic() for m in all_models.values()]++ from letta.settings import model_settings++ if model_settings.auto_mode_enabled and (model_type is None or model_type == "llm"):+ for handle in AUTO_MODE_HANDLES:+ # Generate deterministic 8-char hex ID from handle+ handle_hash = hashlib.sha256(handle.encode()).hexdigest()[:8]+ models.append(+ PydanticProviderModel(+ id=f"model-{handle_hash}",+ handle=handle,+ name=handle.split("/")[1],+ display_name=handle.split("/")[1],+ provider_id="letta",+ model_type="llm",+ model_endpoint_type="openai",+ enabled=True,+ max_context_window=180000,+ supports_token_streaming=True,+ supports_tool_calling=True,+ )+ )++ return models@enforce_types@trace_method@@ -951,6 +979,22 @@ class ProviderManager:NoResultFound: If the handle doesn't exist in the database or BYOK provider"""from letta.orm.errors import NoResultFound+ from letta.settings import model_settings++ # Auto mode handles return a placeholder config for storage/persistence+ if handle in AUTO_MODE_HANDLES:+ if not model_settings.auto_mode_enabled:+ raise NoResultFound(f"Auto mode not enabled for handle='{handle}'")+ return LLMConfig(+ model="auto" if handle == "letta/auto" else "auto-fast",+ model_endpoint_type="openai",+ model_endpoint="",+ context_window=180000,+ handle=handle,+ max_tokens=8192,+ provider_name="letta",+ provider_category=ProviderCategory.base,+ )# Look up the model by handle in the database (for base providers)model = await self.get_model_by_handle_async(handle=handle, actor=actor, model_type="llm")diff --git a/letta/services/streaming_service.py b/letta/services/streaming_service.pyindex c025405e3..5d5030b72 100644--- a/letta/services/streaming_service.py+++ b/letta/services/streaming_service.py@@ -79,6 +79,7 @@ class StreamingService:run_type: str = "streaming",conversation_id: Optional[str] = None,should_lock: bool = False,+ auto_mode_enabled: bool = False,billing_context: "BillingContext | None" = None,) -> tuple[Optional[PydanticRun], Union[StreamingResponse, LettaResponse]]:"""@@ -178,6 +179,7 @@ class StreamingService:lock_key=lock_key, # For lock release (may differ from conversation_id)client_tools=request.client_tools,include_compaction_messages=request.include_compaction_messages,+ auto_mode_enabled=auto_mode_enabled,billing_context=billing_context,)@@ -343,6 +345,7 @@ class StreamingService:lock_key: Optional[str] = None,client_tools: Optional[list[ClientToolSchema]] = None,include_compaction_messages: bool = False,+ auto_mode_enabled: bool = False,billing_context: BillingContext | None = None,) -> AsyncIterator:"""@@ -372,6 +375,7 @@ class StreamingService:conversation_id=conversation_id,client_tools=client_tools,include_compaction_messages=include_compaction_messages,+ auto_mode_enabled=auto_mode_enabled,billing_context=billing_context,)diff --git a/letta/settings.py b/letta/settings.pyindex aa316385f..545bfa511 100644--- a/letta/settings.py+++ b/letta/settings.py@@ -116,6 +116,11 @@ class ModelSettings(BaseSettings):global_max_context_window_limit: int = 128000+ auto_mode_enabled: bool = Field(+ default=False,+ description="Enable auto mode model handles (i.e. letta/auto, letta/auto-fast) in model listing",+ )+inner_thoughts_kwarg: str | None = Field(default=INNER_THOUGHTS_KWARG, description="Key used for passing in inner thoughts.")# env_prefix='my_prefix_'