Platform for stateful agents: AI with advanced memory that can learn and self-improve over time.
feat: allow customizing the handle base for openrouter and for vllm [LET-4609] (#5114)
Charles Packer811b3e6cb67b
diff --git a/letta/schemas/providers/openrouter.py b/letta/schemas/providers/openrouter.pyindex 9e2a3052e..398266450 100644--- a/letta/schemas/providers/openrouter.py+++ b/letta/schemas/providers/openrouter.py@@ -21,6 +21,7 @@ class OpenRouterProvider(OpenAIProvider):provider_category: ProviderCategory = Field(ProviderCategory.base, description="The category of the provider (base or byok)")api_key: str = Field(..., description="API key for the OpenRouter API.")base_url: str = Field("https://openrouter.ai/api/v1", description="Base URL for the OpenRouter API.")+ handle_base: str | None = Field(None, description="Custom handle base name for model handles (e.g., 'custom' instead of 'openrouter').")def _list_llm_models(self, data: list[dict]) -> list[LLMConfig]:"""@@ -33,7 +34,7 @@ class OpenRouterProvider(OpenAIProvider):continuemodel_name, context_window_size = check- handle = self.get_handle(model_name)+ handle = self.get_handle(model_name, base_name=self.handle_base) if self.handle_base else self.get_handle(model_name)config = LLMConfig(model=model_name,diff --git a/letta/schemas/providers/vllm.py b/letta/schemas/providers/vllm.pyindex 0481807e6..f3629e51f 100644--- a/letta/schemas/providers/vllm.py+++ b/letta/schemas/providers/vllm.py@@ -23,6 +23,7 @@ class VLLMProvider(Provider):default_prompt_formatter: str | None = Field(default=None, description="Default prompt formatter (aka model wrapper) to use on a /completions style API.")+ handle_base: str | None = Field(None, description="Custom handle base name for model handles (e.g., 'custom' instead of 'vllm').")async def list_llm_models_async(self) -> list[LLMConfig]:from letta.llm_api.openai import openai_get_model_list_async@@ -43,7 +44,7 @@ class VLLMProvider(Provider):model_endpoint=base_url,model_wrapper=self.default_prompt_formatter,context_window=model["max_model_len"],- handle=self.get_handle(model_name),+ handle=self.get_handle(model_name, base_name=self.handle_base) if self.handle_base else self.get_handle(model_name),provider_name=self.name,provider_category=self.provider_category,)diff --git a/letta/server/server.py b/letta/server/server.pyindex 8b49c3a55..c12fda66b 100644--- a/letta/server/server.py+++ b/letta/server/server.py@@ -278,6 +278,7 @@ class SyncServer(object):name="vllm",base_url=model_settings.vllm_api_base,default_prompt_formatter=model_settings.default_prompt_formatter,+ handle_base=model_settings.vllm_handle_base,))@@ -306,6 +307,7 @@ class SyncServer(object):OpenRouterProvider(name="openrouter",api_key=model_settings.openrouter_api_key,+ handle_base=model_settings.openrouter_handle_base,))diff --git a/letta/settings.py b/letta/settings.pyindex b257f03ad..867331397 100644--- a/letta/settings.py+++ b/letta/settings.py@@ -117,6 +117,7 @@ class ModelSettings(BaseSettings):# See https://openrouter.ai/docs/quick-start for detailsopenrouter_referer: Optional[str] = None # e.g., your site URLopenrouter_title: Optional[str] = None # e.g., your app name+ openrouter_handle_base: Optional[str] = None# deepseekdeepseek_api_key: Optional[str] = None@@ -163,6 +164,7 @@ class ModelSettings(BaseSettings):# vLLMvllm_api_base: Optional[str] = None+ vllm_handle_base: Optional[str] = None# lmstudiolmstudio_base_url: Optional[str] = None