Skip to content

armgit / letta

Public Imported from GitHub
0

Platform for stateful agents: AI with advanced memory that can learn and self-improve over time.

feat(core): Add LLM routing with auto mode and circuit breaker (#9742)

cthomas343ef9096133
diff --git a/letta/agents/base_agent.py b/letta/agents/base_agent.py
index 3e6019a4b..c81f760d4 100644
--- a/letta/agents/base_agent.py
+++ b/letta/agents/base_agent.py
@@ -56,6 +56,7 @@ class BaseAgent(ABC):
input_messages: List[MessageCreate],
max_steps: int = DEFAULT_MAX_STEPS,
run_id: Optional[str] = None,
+ auto_mode_enabled: bool = False,
billing_context: "BillingContext | None" = None,
) -> LettaResponse:
"""
diff --git a/letta/agents/base_agent_v2.py b/letta/agents/base_agent_v2.py
index 515edb76b..0edbee0ce 100644
--- a/letta/agents/base_agent_v2.py
+++ b/letta/agents/base_agent_v2.py
@@ -53,6 +53,7 @@ class BaseAgentV2(ABC):
request_start_timestamp_ns: int | None = None,
client_tools: list["ClientToolSchema"] | None = None,
include_compaction_messages: bool = False, # Not used in V2, but accepted for API compatibility
+ auto_mode_enabled: bool = False, # Not used in V2, but accepted for API compatibility
billing_context: "BillingContext | None" = None,
) -> LettaResponse:
"""
@@ -78,6 +79,7 @@ class BaseAgentV2(ABC):
conversation_id: str | None = None,
client_tools: list["ClientToolSchema"] | None = None,
include_compaction_messages: bool = False, # Not used in V2, but accepted for API compatibility
+ auto_mode_enabled: bool = False, # Not used in V2, but accepted for API compatibility
billing_context: "BillingContext | None" = None,
) -> AsyncGenerator[LettaMessage | LegacyLettaMessage | MessageStreamStatus, None]:
"""
diff --git a/letta/agents/letta_agent_v2.py b/letta/agents/letta_agent_v2.py
index 13fb9b07f..b958c9fde 100644
--- a/letta/agents/letta_agent_v2.py
+++ b/letta/agents/letta_agent_v2.py
@@ -186,6 +186,7 @@ class LettaAgentV2(BaseAgentV2):
request_start_timestamp_ns: int | None = None,
client_tools: list[ClientToolSchema] | None = None,
include_compaction_messages: bool = False, # Not used in V2, but accepted for API compatibility
+ auto_mode_enabled: bool = False, # Not used in V2, but accepted for API compatibility
billing_context: "BillingContext | None" = None,
) -> LettaResponse:
"""
@@ -292,6 +293,7 @@ class LettaAgentV2(BaseAgentV2):
conversation_id: str | None = None, # Not used in V2, but accepted for API compatibility
client_tools: list[ClientToolSchema] | None = None,
include_compaction_messages: bool = False, # Not used in V2, but accepted for API compatibility
+ auto_mode_enabled: bool = False, # Not used in V2, but accepted for API compatibility
billing_context: BillingContext | None = None,
) -> AsyncGenerator[str, None]:
"""
diff --git a/letta/agents/letta_agent_v3.py b/letta/agents/letta_agent_v3.py
index 3a77c011e..baa159275 100644
--- a/letta/agents/letta_agent_v3.py
+++ b/letta/agents/letta_agent_v3.py
@@ -21,10 +21,19 @@ from letta.agents.helpers import (
)
from letta.agents.letta_agent_v2 import LettaAgentV2
from letta.constants import DEFAULT_MAX_STEPS, NON_USER_MSG_PREFIX, REQUEST_HEARTBEAT_PARAM
-from letta.errors import ContextWindowExceededError, LLMEmptyResponseError, LLMError, SystemPromptTokenExceededError
+from letta.errors import (
+ ContextWindowExceededError,
+ LLMEmptyResponseError,
+ LLMError,
+ LLMProviderOverloaded,
+ LLMRateLimitError,
+ LLMServerError,
+ SystemPromptTokenExceededError,
+)
from letta.helpers import ToolRulesSolver
from letta.helpers.datetime_helpers import get_utc_time, get_utc_timestamp_ns
from letta.helpers.tool_execution_helper import enable_strict_mode
+from letta.llm_api.llm_client import LLMClient
from letta.local_llm.constants import INNER_THOUGHTS_KWARG
from letta.otel.tracing import trace_method
from letta.schemas.agent import AgentState
@@ -58,6 +67,8 @@ from letta.server.rest_api.utils import (
)
from letta.services.conversation_manager import ConversationManager
from letta.services.helpers.tool_parser_helper import runtime_override_tool_json_schema
+from letta.services.llm_router import get_llm_routing_client
+from letta.services.provider_manager import AUTO_MODE_HANDLES
from letta.services.summarizer.compact import compact_messages
from letta.services.summarizer.summarizer_config import CompactionSettings
from letta.services.summarizer.summarizer_sliding_window import count_tokens
@@ -150,6 +161,7 @@ class LettaAgentV3(LettaAgentV2):
conversation_id: str | None = None,
client_tools: list[ClientToolSchema] | None = None,
include_compaction_messages: bool = False,
+ auto_mode_enabled: bool = False,
billing_context: "BillingContext | None" = None,
) -> LettaResponse:
"""
@@ -261,6 +273,7 @@ class LettaAgentV3(LettaAgentV2):
include_return_message_types=include_return_message_types,
request_start_timestamp_ns=request_start_timestamp_ns,
include_compaction_messages=include_compaction_messages,
+ auto_mode_enabled=auto_mode_enabled,
)
input_messages_to_persist = [] # clear after first step
@@ -365,6 +378,7 @@ class LettaAgentV3(LettaAgentV2):
conversation_id: str | None = None,
client_tools: list[ClientToolSchema] | None = None,
include_compaction_messages: bool = False,
+ auto_mode_enabled: bool = False,
billing_context: BillingContext | None = None,
) -> AsyncGenerator[str, None]:
"""
@@ -493,6 +507,7 @@ class LettaAgentV3(LettaAgentV2):
include_return_message_types=include_return_message_types,
request_start_timestamp_ns=request_start_timestamp_ns,
include_compaction_messages=include_compaction_messages,
+ auto_mode_enabled=auto_mode_enabled,
)
input_messages_to_persist = [] # clear after first step
async for chunk in response:
@@ -792,6 +807,7 @@ class LettaAgentV3(LettaAgentV2):
dry_run: bool = False,
enforce_run_id_set: bool = True,
include_compaction_messages: bool = False,
+ auto_mode_enabled: bool = False,
) -> AsyncGenerator[LettaMessage | dict, None]:
"""
Execute a single agent step (one LLM call and tool execution).
@@ -923,13 +939,37 @@ class LettaAgentV3(LettaAgentV2):
step_id=step_id, run_id=run_id
)
+ # Auto mode: resolve handle to actual model config
+ auto_mode_handle = self.agent_state.llm_config.handle
+ is_auto_mode = auto_mode_handle in AUTO_MODE_HANDLES
+ is_primary = False
+ primary_handle = ""
+
+ if is_auto_mode:
+ routing_client = await get_llm_routing_client()
+ active_llm_config, is_primary, primary_handle = await routing_client.resolve_auto_mode_config(
+ auto_mode_enabled=auto_mode_enabled,
+ stored_llm_config=self.agent_state.llm_config,
+ actor=self.actor,
+ )
+ if not is_primary:
+ self.logger.info(f"[AUTO MODE]: primary {primary_handle} rerouted, falling back to {active_llm_config.handle}")
+ active_llm_client = LLMClient.create(
+ provider_type=active_llm_config.model_endpoint_type,
+ put_inner_thoughts_first=True,
+ actor=self.actor,
+ )
+ else:
+ active_llm_config = self.agent_state.llm_config
+ active_llm_client = self.llm_client
+
force_tool_call = valid_tools[0]["name"] if len(valid_tools) == 1 and self._require_tool_call else None
for llm_request_attempt in range(summarizer_settings.max_summarizer_retries + 1):
try:
- request_data = self.llm_client.build_request_data(
+ request_data = active_llm_client.build_request_data(
agent_type=self.agent_state.agent_type,
messages=messages,
- llm_config=self.agent_state.llm_config,
+ llm_config=active_llm_config,
tools=valid_tools,
force_tool_call=force_tool_call,
requires_subsequent_tool_call=self._require_tool_call,
@@ -945,30 +985,30 @@ class LettaAgentV3(LettaAgentV2):
)
# Anthropic/Bedrock/MiniMax parallel tool use (MiniMax uses Anthropic-compatible API)
- if self.agent_state.llm_config.model_endpoint_type in ["anthropic", "bedrock", "minimax"]:
+ if active_llm_config.model_endpoint_type in ["anthropic", "bedrock", "minimax"]:
if (
isinstance(request_data.get("tool_choice"), dict)
and "disable_parallel_tool_use" in request_data["tool_choice"]
):
# Gate parallel tool use on both: no tool rules and toggled on
- if no_tool_rules and self.agent_state.llm_config.parallel_tool_calls:
+ if no_tool_rules and active_llm_config.parallel_tool_calls:
request_data["tool_choice"]["disable_parallel_tool_use"] = False
else:
# Explicitly disable when tool rules present or llm_config toggled off
request_data["tool_choice"]["disable_parallel_tool_use"] = True
# OpenAI parallel tool use
- elif self.agent_state.llm_config.model_endpoint_type == "openai":
+ elif active_llm_config.model_endpoint_type == "openai":
# For OpenAI, we control parallel tool calling via parallel_tool_calls field
# Only allow parallel tool calls when no tool rules and enabled in config
if "parallel_tool_calls" in request_data:
- if no_tool_rules and self.agent_state.llm_config.parallel_tool_calls:
+ if no_tool_rules and active_llm_config.parallel_tool_calls:
request_data["parallel_tool_calls"] = True
else:
request_data["parallel_tool_calls"] = False
# Gemini (Google AI/Vertex) parallel tool use
- elif self.agent_state.llm_config.model_endpoint_type in ["google_ai", "google_vertex"]:
+ elif active_llm_config.model_endpoint_type in ["google_ai", "google_vertex"]:
# Gemini supports parallel tool calling natively through multiple parts in the response
# We just need to ensure the config flag is set for tracking purposes
# The actual handling happens in GoogleVertexClient.convert_response_to_chat_completion
@@ -997,6 +1037,10 @@ class LettaAgentV3(LettaAgentV2):
if llm_adapter.supports_token_streaming():
if include_return_message_types is None or chunk.message_type in include_return_message_types:
yield chunk
+ # Report success to circuit breaker
+ if is_auto_mode and is_primary:
+ routing_client = await get_llm_routing_client()
+ await routing_client.record_success(primary_handle)
# If you've reached this point without an error, break out of retry loop
break
except ValueError as e:
@@ -1005,6 +1049,62 @@ class LettaAgentV3(LettaAgentV2):
except LLMEmptyResponseError as e:
self.stop_reason = LettaStopReason(stop_reason=StopReasonType.invalid_llm_response.value)
raise e
+ except (LLMRateLimitError, LLMServerError, LLMProviderOverloaded) as e:
+ if is_auto_mode and is_primary:
+ routing_client = await get_llm_routing_client()
+ await routing_client.record_failure(primary_handle)
+
+ fallback_config = await routing_client.get_fallback_config(
+ stored_llm_config=self.agent_state.llm_config,
+ actor=self.actor,
+ )
+ self.logger.warning(
+ f"[AUTO MODE]: primary {primary_handle} failed ({type(e).__name__}), falling back to {fallback_config.handle}"
+ )
+ fallback_client = LLMClient.create(
+ provider_type=fallback_config.model_endpoint_type,
+ put_inner_thoughts_first=True,
+ actor=self.actor,
+ )
+ fallback_request_data = fallback_client.build_request_data(
+ agent_type=self.agent_state.agent_type,
+ messages=messages,
+ llm_config=fallback_config,
+ tools=valid_tools,
+ force_tool_call=force_tool_call,
+ requires_subsequent_tool_call=self._require_tool_call,
+ tool_return_truncation_chars=self._compute_tool_return_truncation_chars(),
+ )
+ fallback_adapter = SimpleLLMRequestAdapter(
+ llm_client=fallback_client,
+ llm_config=fallback_config,
+ call_type=LLMCallType.agent_step,
+ agent_id=self.agent_state.id,
+ agent_tags=self.agent_state.tags,
+ run_id=run_id,
+ org_id=self.actor.organization_id,
+ user_id=self.actor.id,
+ )
+ fallback_invocation = fallback_adapter.invoke_llm(
+ request_data=fallback_request_data,
+ messages=messages,
+ tools=valid_tools,
+ use_assistant_message=False,
+ requires_approval_tools=self.tool_rules_solver.get_requires_approval_tools(
+ set([t["name"] for t in valid_tools])
+ )
+ + [ct.name for ct in self.client_tools],
+ step_id=step_id,
+ actor=self.actor,
+ )
+ async for chunk in fallback_invocation:
+ if fallback_adapter.supports_token_streaming():
+ if include_return_message_types is None or chunk.message_type in include_return_message_types:
+ yield chunk
+ llm_adapter = fallback_adapter
+ break
+ self.stop_reason = LettaStopReason(stop_reason=StopReasonType.llm_api_error.value)
+ raise e
except LLMError as e:
self.stop_reason = LettaStopReason(stop_reason=StopReasonType.llm_api_error.value)
raise e
diff --git a/letta/groups/sleeptime_multi_agent_v2.py b/letta/groups/sleeptime_multi_agent_v2.py
index e842e0b7f..fb833b8d8 100644
--- a/letta/groups/sleeptime_multi_agent_v2.py
+++ b/letta/groups/sleeptime_multi_agent_v2.py
@@ -70,6 +70,7 @@ class SleeptimeMultiAgentV2(BaseAgent):
use_assistant_message: bool = True,
request_start_timestamp_ns: int | None = None,
include_return_message_types: list[MessageType] | None = None,
+ auto_mode_enabled: bool = False, # Not used in multi-agent, but accepted for API compatibility
billing_context: "BillingContext | None" = None,
) -> LettaResponse:
run_ids = []
diff --git a/letta/groups/sleeptime_multi_agent_v3.py b/letta/groups/sleeptime_multi_agent_v3.py
index 7a148eada..0192e9b1d 100644
--- a/letta/groups/sleeptime_multi_agent_v3.py
+++ b/letta/groups/sleeptime_multi_agent_v3.py
@@ -48,6 +48,7 @@ class SleeptimeMultiAgentV3(LettaAgentV2):
request_start_timestamp_ns: int | None = None,
client_tools: list[ClientToolSchema] | None = None,
include_compaction_messages: bool = False,
+ auto_mode_enabled: bool = False, # Not used in multi-agent, but accepted for API compatibility
billing_context: "BillingContext | None" = None,
) -> LettaResponse:
self.run_ids = []
diff --git a/letta/groups/sleeptime_multi_agent_v4.py b/letta/groups/sleeptime_multi_agent_v4.py
index 150cd3f7f..782094eae 100644
--- a/letta/groups/sleeptime_multi_agent_v4.py
+++ b/letta/groups/sleeptime_multi_agent_v4.py
@@ -48,6 +48,7 @@ class SleeptimeMultiAgentV4(LettaAgentV3):
conversation_id: str | None = None,
client_tools: list[ClientToolSchema] | None = None,
include_compaction_messages: bool = False,
+ auto_mode_enabled: bool = False, # Not used in multi-agent, but accepted for API compatibility
billing_context: "BillingContext | None" = None,
) -> LettaResponse:
self.run_ids = []
diff --git a/letta/server/rest_api/dependencies.py b/letta/server/rest_api/dependencies.py
index 66ea43bb4..15772d754 100644
--- a/letta/server/rest_api/dependencies.py
+++ b/letta/server/rest_api/dependencies.py
@@ -20,6 +20,7 @@ class ExperimentalParams(BaseModel):
letta_v1_agent: Optional[bool] = None
letta_v1_agent_message_async: Optional[bool] = None
modal_sandbox: Optional[bool] = None
+ auto_mode: Optional[bool] = None
class HeaderParams(BaseModel):
@@ -46,6 +47,7 @@ def get_headers(
None, alias="X-Experimental-Letta-V1-Agent-Message-Async", include_in_schema=False
),
modal_sandbox: Optional[str] = Header(None, alias="X-Experimental-Modal-Sandbox", include_in_schema=False),
+ auto_mode: Optional[str] = Header(None, alias="X-Experimental-Auto-Mode", include_in_schema=False),
billing_plan_type: Optional[str] = Header(None, alias="X-Billing-Plan-Type", include_in_schema=False),
billing_cost_source: Optional[str] = Header(None, alias="X-Billing-Cost-Source", include_in_schema=False),
billing_customer_id: Optional[str] = Header(None, alias="X-Billing-Customer-Id", include_in_schema=False),
@@ -69,6 +71,7 @@ def get_headers(
letta_v1_agent=(letta_v1_agent == "true") if letta_v1_agent else None,
letta_v1_agent_message_async=(letta_v1_agent_message_async == "true") if letta_v1_agent_message_async else None,
modal_sandbox=(modal_sandbox == "true") if modal_sandbox else None,
+ auto_mode=(auto_mode == "true") if auto_mode else None,
),
billing_context=BillingContext(
plan_type=billing_plan_type,
diff --git a/letta/server/rest_api/routers/v1/agents.py b/letta/server/rest_api/routers/v1/agents.py
index dce472409..962728d1b 100644
--- a/letta/server/rest_api/routers/v1/agents.py
+++ b/letta/server/rest_api/routers/v1/agents.py
@@ -1692,12 +1692,14 @@ async def send_message(
actor = await server.user_manager.get_actor_or_default_async(actor_id=headers.actor_id)
if request.streaming and is_1_0_sdk:
+ auto_mode_enabled = bool(headers.experimental_params and headers.experimental_params.auto_mode)
streaming_service = StreamingService(server)
run, result = await streaming_service.create_agent_stream(
agent_id=agent_id,
actor=actor,
request=request,
run_type="send_message",
+ auto_mode_enabled=auto_mode_enabled,
billing_context=headers.billing_context,
)
return result
@@ -1759,6 +1761,7 @@ async def send_message(
}
)
+ auto_mode_enabled = bool(headers.experimental_params and headers.experimental_params.auto_mode)
agent_loop = AgentLoop.load(agent_state=agent, actor=actor)
result = await agent_loop.step(
request.messages,
@@ -1769,6 +1772,7 @@ async def send_message(
include_return_message_types=request.include_return_message_types,
client_tools=request.client_tools,
include_compaction_messages=request.include_compaction_messages,
+ auto_mode_enabled=auto_mode_enabled,
billing_context=headers.billing_context,
)
run_status = result.stop_reason.stop_reason.run_status
diff --git a/letta/server/server.py b/letta/server/server.py
index ea6eef31b..23e2bf62e 100644
--- a/letta/server/server.py
+++ b/letta/server/server.py
@@ -1240,9 +1240,25 @@ class SyncServer(object):
)
# Build LLMConfig objects from database
+ from letta.services.provider_manager import AUTO_MODE_HANDLES
+
provider_cache: Dict[str, Provider] = {}
typed_provider_cache: Dict[str, Any] = {}
for model in provider_models:
+ # Handle synthetic auto mode models separately
+ if model.handle in AUTO_MODE_HANDLES:
+ llm_config = LLMConfig(
+ model=model.name,
+ model_endpoint_type=model.model_endpoint_type,
+ model_endpoint="",
+ context_window=model.max_context_window or 180000,
+ handle=model.handle,
+ provider_name="letta",
+ max_tokens=8192,
+ )
+ llm_models.append(llm_config)
+ continue
+
# Get provider details (with caching to avoid N+1 queries)
if model.provider_id not in provider_cache:
provider_cache[model.provider_id] = await self.provider_manager.get_provider_async(model.provider_id, actor)
diff --git a/letta/services/llm_router/__init__.py b/letta/services/llm_router/__init__.py
new file mode 100644
index 000000000..813f6f93e
--- /dev/null
+++ b/letta/services/llm_router/__init__.py
@@ -0,0 +1,18 @@
+"""LLM routing client with circuit breaker and auto mode support.
+
+Conditionally imports Redis-backed implementation or falls back to noop base.
+"""
+
+try:
+ from .client import LLMRoutingClient, get_llm_routing_client
+except ImportError:
+ from .client_base import LLMRoutingClient
+
+ async def get_llm_routing_client():
+ return LLMRoutingClient()
+
+
+__all__ = [
+ "LLMRoutingClient",
+ "get_llm_routing_client",
+]
diff --git a/letta/services/llm_router/client.py b/letta/services/llm_router/client.py
new file mode 100644
index 000000000..e7659f1aa
--- /dev/null
+++ b/letta/services/llm_router/client.py
@@ -0,0 +1,212 @@
+"""Redis-backed LLM routing client for model health tracking and auto mode resolution.
+
+Tracks overload errors per model handle using Redis sorted sets for
+sliding window error counting. Works across multiple pods.
+
+The noop base class lives in client_base.py.
+"""
+
+import time
+from enum import Enum
+from typing import TYPE_CHECKING, Optional
+
+from letta.data_sources.redis_client import AsyncRedisClient
+from letta.log import get_logger
+
+from .client_base import LLMRoutingClient as LLMRoutingClientBase
+
+if TYPE_CHECKING:
+ from letta.schemas.llm_config import LLMConfig
+ from letta.schemas.user import User
+
+logger = get_logger(__name__)
+
+# Auto mode model routing
+AUTO_MODE_PRIMARY = "zai/glm-5"
+AUTO_MODE_FALLBACK = "zai/glm-5"
+
+# Circuit breaker configuration
+ERROR_WINDOW_SECONDS = 60 # Sliding window for error counting
+ERROR_THRESHOLD = 1 # Errors needed to trip circuit
+OPEN_DURATION_SECONDS = 30 # How long circuit stays open
+
+# Redis key prefixes
+CIRCUIT_ERRORS_KEY = "circuit:errors:"
+CIRCUIT_STATE_KEY = "circuit:state:"
+CIRCUIT_OPENED_AT_KEY = "circuit:opened:"
+
+
+class CircuitState(str, Enum):
+ CLOSED = "CLOSED"
+ OPEN = "OPEN"
+
+
+class LLMRoutingClient(LLMRoutingClientBase):
+ """Redis-backed LLM routing client for model health tracking and auto mode routing.
+
+ Uses an internal circuit breaker (Redis sorted sets) for sliding window error
+ counting across pods. The circuit breaker is an implementation detail.
+ """
+
+ def __init__(self, redis_client: AsyncRedisClient):
+ self.redis = redis_client
+
+ async def resolve_auto_mode_config(
+ self,
+ auto_mode_enabled: bool,
+ stored_llm_config: "LLMConfig",
+ actor: "User",
+ ) -> tuple["LLMConfig", bool, str]:
+ """Resolve an auto mode handle to an actual model config.
+
+ Uses the internal circuit breaker to decide between primary and fallback.
+ Preserves user-configured context_window and max_tokens from the stored config.
+
+ Args:
+ auto_mode_enabled: Whether the experimental auto_mode header was set.
+ stored_llm_config: The agent's stored LLM config (with auto mode handle).
+ actor: The user actor for provider lookups.
+
+ Returns:
+ Tuple of (resolved_config, is_primary, primary_handle).
+ """
+ from letta.services.provider_manager import ProviderManager
+
+ provider_manager = ProviderManager()
+ primary_handle = AUTO_MODE_PRIMARY
+
+ if auto_mode_enabled:
+ if await self._is_healthy(primary_handle):
+ config = await provider_manager.get_llm_config_from_handle(primary_handle, actor)
+ config = config.model_copy(
+ update={
+ "context_window": min(stored_llm_config.context_window, config.context_window),
+ "max_tokens": min(stored_llm_config.max_tokens, config.max_tokens),
+ }
+ )
+ return config, True, primary_handle
+
+ logger.warning(f"Auto mode: primary {primary_handle} unhealthy, using {AUTO_MODE_FALLBACK}")
+
+ config = await provider_manager.get_llm_config_from_handle(AUTO_MODE_FALLBACK, actor)
+ config = config.model_copy(
+ update={
+ "context_window": min(stored_llm_config.context_window, config.context_window),
+ "max_tokens": min(stored_llm_config.max_tokens, config.max_tokens),
+ }
+ )
+ return config, False, primary_handle
+
+ async def record_failure(self, handle: str) -> None:
+ """Record a failure for a model handle. May trip the circuit.
+
+ Args:
+ handle: The model handle that failed.
+ """
+ client = await self.redis.get_client()
+ now = time.time()
+ errors_key = f"{CIRCUIT_ERRORS_KEY}{handle}"
+
+ await client.zadd(errors_key, {str(now): now})
+ await client.zremrangebyscore(errors_key, 0, now - ERROR_WINDOW_SECONDS)
+ await client.expire(errors_key, ERROR_WINDOW_SECONDS * 2)
+
+ error_count = await client.zcard(errors_key)
+ if error_count >= ERROR_THRESHOLD:
+ await self._open_circuit(client, handle)
+
+ async def record_success(self, handle: str) -> None:
+ """Record a success for a model handle. Closes circuit if open.
+
+ Args:
+ handle: The model handle that succeeded.
+ """
+ state = await self._get_state(handle)
+ if state == CircuitState.OPEN:
+ await self._close_circuit(handle)
+
+ async def get_fallback_config(
+ self,
+ stored_llm_config: "LLMConfig",
+ actor: "User",
+ ) -> "LLMConfig":
+ """Get the fallback config for auto mode.
+
+ Args:
+ stored_llm_config: The agent's stored LLM config (with auto mode handle).
+ actor: The user actor for provider lookups.
+
+ Returns:
+ The fallback LLM config with preserved context_window and max_tokens.
+ """
+ from letta.services.provider_manager import ProviderManager
+
+ provider_manager = ProviderManager()
+ fallback_config = await provider_manager.get_llm_config_from_handle(AUTO_MODE_FALLBACK, actor)
+ fallback_config = fallback_config.model_copy(
+ update={
+ "context_window": min(stored_llm_config.context_window, fallback_config.context_window),
+ "max_tokens": min(stored_llm_config.max_tokens, fallback_config.max_tokens),
+ }
+ )
+ return fallback_config
+
+ async def _is_healthy(self, handle: str) -> bool:
+ """Check if a model handle is healthy (internal circuit breaker).
+
+ Returns True if the circuit is closed, or if the open duration has elapsed
+ (allowing a retry probe).
+
+ Args:
+ handle: The model handle to check.
+
+ Returns:
+ True if the model should be tried, False if it should be skipped.
+ """
+ state = await self._get_state(handle)
+ if state == CircuitState.CLOSED:
+ return True
+
+ opened_at = await self._get_opened_at(handle)
+ if opened_at and time.time() - opened_at >= OPEN_DURATION_SECONDS:
+ return True
+
+ return False
+
+ async def _get_state(self, handle: str) -> CircuitState:
+ client = await self.redis.get_client()
+ state = await client.get(f"{CIRCUIT_STATE_KEY}{handle}")
+ return CircuitState.OPEN if state == CircuitState.OPEN.value else CircuitState.CLOSED
+
+ async def _get_opened_at(self, handle: str) -> Optional[float]:
+ client = await self.redis.get_client()
+ val = await client.get(f"{CIRCUIT_OPENED_AT_KEY}{handle}")
+ return float(val) if val else None
+
+ async def _open_circuit(self, client, handle: str) -> None:
+ now = time.time()
+ await client.set(f"{CIRCUIT_STATE_KEY}{handle}", CircuitState.OPEN.value, ex=OPEN_DURATION_SECONDS * 2)
+ await client.set(f"{CIRCUIT_OPENED_AT_KEY}{handle}", str(now), ex=OPEN_DURATION_SECONDS * 2)
+ logger.warning(f"Circuit OPENED for model {handle}")
+
+ async def _close_circuit(self, handle: str) -> None:
+ client = await self.redis.get_client()
+ await client.delete(f"{CIRCUIT_STATE_KEY}{handle}", f"{CIRCUIT_OPENED_AT_KEY}{handle}", f"{CIRCUIT_ERRORS_KEY}{handle}")
+ logger.info(f"Circuit CLOSED for model {handle}")
+
+
+async def get_llm_routing_client() -> LLMRoutingClientBase:
+ """Factory that returns the appropriate LLM routing client instance.
+
+ Returns the Redis-backed implementation if Redis is available,
+ otherwise returns the noop base.
+
+ Returns:
+ An LLMRoutingClient instance.
+ """
+ from letta.data_sources.redis_client import NoopAsyncRedisClient, get_redis_client
+
+ redis_client = await get_redis_client()
+ if isinstance(redis_client, NoopAsyncRedisClient):
+ return LLMRoutingClientBase()
+ return LLMRoutingClient(redis_client)
diff --git a/letta/services/llm_router/client_base.py b/letta/services/llm_router/client_base.py
new file mode 100644
index 000000000..3c69c8d01
--- /dev/null
+++ b/letta/services/llm_router/client_base.py
@@ -0,0 +1,69 @@
+"""Base LLM routing client.
+
+This is the base (OSS/self-hosted) implementation without Redis support.
+The Redis-backed implementation lives in client.py.
+"""
+
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+ from letta.schemas.llm_config import LLMConfig
+ from letta.schemas.user import User
+
+
+class LLMRoutingClient:
+ """Base LLM routing client.
+
+ Used when Redis is not configured (OSS/self-hosted).
+ Auto mode is not supported without Redis.
+ """
+
+ async def resolve_auto_mode_config(
+ self,
+ auto_mode_enabled: bool,
+ stored_llm_config: "LLMConfig",
+ actor: "User",
+ ) -> tuple["LLMConfig", bool, str]:
+ """Resolve an auto mode handle to an actual model config.
+
+ Args:
+ auto_mode_enabled: Whether the experimental auto_mode header was set.
+ stored_llm_config: The agent's stored LLM config (with auto mode handle).
+ actor: The user actor for provider lookups.
+
+ Returns:
+ Tuple of (resolved_config, is_primary, primary_handle).
+
+ Raises:
+ RuntimeError: Auto mode requires Redis for circuit breaker support.
+ """
+ raise RuntimeError(
+ "Auto mode requires Redis for circuit breaker support. Configure Redis or disable auto_mode_enabled in settings."
+ )
+
+ async def record_failure(self, handle: str) -> None:
+ """Record a failure for a model handle (noop in base)."""
+
+ async def record_success(self, handle: str) -> None:
+ """Record a success for a model handle (noop in base)."""
+
+ async def get_fallback_config(
+ self,
+ stored_llm_config: "LLMConfig",
+ actor: "User",
+ ) -> "LLMConfig":
+ """Get the fallback config for auto mode.
+
+ Args:
+ stored_llm_config: The agent's stored LLM config.
+ actor: The user actor for provider lookups.
+
+ Returns:
+ The fallback LLM config.
+
+ Raises:
+ RuntimeError: Auto mode requires Redis for circuit breaker support.
+ """
+ raise RuntimeError(
+ "Auto mode requires Redis for circuit breaker support. Configure Redis or disable auto_mode_enabled in settings."
+ )
diff --git a/letta/services/provider_manager.py b/letta/services/provider_manager.py
index d1c98dc52..2c8359488 100644
--- a/letta/services/provider_manager.py
+++ b/letta/services/provider_manager.py
@@ -1,3 +1,4 @@
+import hashlib
from typing import List, Optional, Tuple, Union
from sqlalchemy import and_, select
@@ -19,6 +20,9 @@ from letta.validators import raise_on_invalid_id
logger = get_logger(__name__)
+# Auto mode model handles
+AUTO_MODE_HANDLES = ["letta/auto", "letta/auto-fast"]
+
class ProviderManager:
@enforce_types
@@ -929,7 +933,31 @@ class ProviderManager:
all_models = {(m.handle, m.model_type): m for m in global_models}
all_models.update({(m.handle, m.model_type): m for m in org_models})
- return [m.to_pydantic() for m in all_models.values()]
+ models = [m.to_pydantic() for m in all_models.values()]
+
+ from letta.settings import model_settings
+
+ if model_settings.auto_mode_enabled and (model_type is None or model_type == "llm"):
+ for handle in AUTO_MODE_HANDLES:
+ # Generate deterministic 8-char hex ID from handle
+ handle_hash = hashlib.sha256(handle.encode()).hexdigest()[:8]
+ models.append(
+ PydanticProviderModel(
+ id=f"model-{handle_hash}",
+ handle=handle,
+ name=handle.split("/")[1],
+ display_name=handle.split("/")[1],
+ provider_id="letta",
+ model_type="llm",
+ model_endpoint_type="openai",
+ enabled=True,
+ max_context_window=180000,
+ supports_token_streaming=True,
+ supports_tool_calling=True,
+ )
+ )
+
+ return models
@enforce_types
@trace_method
@@ -951,6 +979,22 @@ class ProviderManager:
NoResultFound: If the handle doesn't exist in the database or BYOK provider
"""
from letta.orm.errors import NoResultFound
+ from letta.settings import model_settings
+
+ # Auto mode handles return a placeholder config for storage/persistence
+ if handle in AUTO_MODE_HANDLES:
+ if not model_settings.auto_mode_enabled:
+ raise NoResultFound(f"Auto mode not enabled for handle='{handle}'")
+ return LLMConfig(
+ model="auto" if handle == "letta/auto" else "auto-fast",
+ model_endpoint_type="openai",
+ model_endpoint="",
+ context_window=180000,
+ handle=handle,
+ max_tokens=8192,
+ provider_name="letta",
+ provider_category=ProviderCategory.base,
+ )
# Look up the model by handle in the database (for base providers)
model = await self.get_model_by_handle_async(handle=handle, actor=actor, model_type="llm")
diff --git a/letta/services/streaming_service.py b/letta/services/streaming_service.py
index c025405e3..5d5030b72 100644
--- a/letta/services/streaming_service.py
+++ b/letta/services/streaming_service.py
@@ -79,6 +79,7 @@ class StreamingService:
run_type: str = "streaming",
conversation_id: Optional[str] = None,
should_lock: bool = False,
+ auto_mode_enabled: bool = False,
billing_context: "BillingContext | None" = None,
) -> tuple[Optional[PydanticRun], Union[StreamingResponse, LettaResponse]]:
"""
@@ -178,6 +179,7 @@ class StreamingService:
lock_key=lock_key, # For lock release (may differ from conversation_id)
client_tools=request.client_tools,
include_compaction_messages=request.include_compaction_messages,
+ auto_mode_enabled=auto_mode_enabled,
billing_context=billing_context,
)
@@ -343,6 +345,7 @@ class StreamingService:
lock_key: Optional[str] = None,
client_tools: Optional[list[ClientToolSchema]] = None,
include_compaction_messages: bool = False,
+ auto_mode_enabled: bool = False,
billing_context: BillingContext | None = None,
) -> AsyncIterator:
"""
@@ -372,6 +375,7 @@ class StreamingService:
conversation_id=conversation_id,
client_tools=client_tools,
include_compaction_messages=include_compaction_messages,
+ auto_mode_enabled=auto_mode_enabled,
billing_context=billing_context,
)
diff --git a/letta/settings.py b/letta/settings.py
index aa316385f..545bfa511 100644
--- a/letta/settings.py
+++ b/letta/settings.py
@@ -116,6 +116,11 @@ class ModelSettings(BaseSettings):
global_max_context_window_limit: int = 128000
+ auto_mode_enabled: bool = Field(
+ default=False,
+ description="Enable auto mode model handles (i.e. letta/auto, letta/auto-fast) in model listing",
+ )
+
inner_thoughts_kwarg: str | None = Field(default=INNER_THOUGHTS_KWARG, description="Key used for passing in inner thoughts.")
# env_prefix='my_prefix_'