Platform for stateful agents: AI with advanced memory that can learn and self-improve over time.
fix: lmstudio support for qwen and llama
jnjpngde3c99d7f520
diff --git a/letta/llm_api/openai_client.py b/letta/llm_api/openai_client.pyindex c9561cc2f..84675f6ce 100644--- a/letta/llm_api/openai_client.py+++ b/letta/llm_api/openai_client.py@@ -101,6 +101,8 @@ def requires_auto_tool_choice(llm_config: LLMConfig) -> bool:return Trueif llm_config.handle and "vllm" in llm_config.handle:return True+ if llm_config.compatibility_type == "mlx":+ return Truereturn Falsediff --git a/letta/schemas/llm_config.py b/letta/schemas/llm_config.pyindex cccf93b06..e2677334b 100644--- a/letta/schemas/llm_config.py+++ b/letta/schemas/llm_config.py@@ -81,6 +81,7 @@ class LLMConfig(BaseModel):None, # Can also deafult to 0.0?description="Positive values penalize new tokens based on their existing frequency in the text so far, decreasing the model's likelihood to repeat the same line verbatim. From OpenAI: Number between -2.0 and 2.0.",)+ compatibility_type: Optional[Literal["gguf", "mlx"]] = Field(None, description="The framework compatibility type for the model.")# FIXME hack to silence pydantic protected namespace warningmodel_config = ConfigDict(protected_namespaces=())diff --git a/letta/schemas/providers/lmstudio.py b/letta/schemas/providers/lmstudio.pyindex 6e4a639ab..910e37da0 100644--- a/letta/schemas/providers/lmstudio.py+++ b/letta/schemas/providers/lmstudio.py@@ -45,6 +45,12 @@ class LMStudioOpenAIProvider(OpenAIProvider):continuemodel_name, context_window_size = check+ if "compatibility_type" in model:+ compatibility_type = model["compatibility_type"]+ else:+ warnings.warn(f"LMStudio OpenAI model missing 'compatibility_type' field: {model}")+ continue+configs.append(LLMConfig(model=model_name,@@ -52,6 +58,7 @@ class LMStudioOpenAIProvider(OpenAIProvider):model_endpoint=self.base_url,context_window=context_window_size,handle=self.get_handle(model_name),+ compatibility_type=compatibility_type,provider_name=self.name,provider_category=self.provider_category,)diff --git a/letta/services/helpers/agent_manager_helper.py b/letta/services/helpers/agent_manager_helper.pyindex ae13435fd..ccebf35bb 100644--- a/letta/services/helpers/agent_manager_helper.py+++ b/letta/services/helpers/agent_manager_helper.py@@ -1,4 +1,5 @@import os+import uuidfrom datetime import datetimefrom typing import List, Literal, Optional, Set@@ -351,21 +352,40 @@ def initialize_message_sequence(first_user_message = get_login_event(agent_state.timezone) # event letting Letta know the user just logged inif include_initial_boot_message:+ llm_config = agent_state.llm_config+ uuid_str = str(uuid.uuid4())++ # Some LMStudio models (e.g. ministral) require the tool call ID to be 9 alphanumeric characters+ tool_call_id = uuid_str[:9] if llm_config.provider_name == "lmstudio_openai" else uuid_str+if agent_state.agent_type == AgentType.sleeptime_agent:initial_boot_messages = []- elif agent_state.llm_config.model is not None and "gpt-3.5" in agent_state.llm_config.model:- initial_boot_messages = get_initial_boot_messages("startup_with_send_message_gpt35", agent_state.timezone)+ elif llm_config.model is not None and "gpt-3.5" in llm_config.model:+ initial_boot_messages = get_initial_boot_messages("startup_with_send_message_gpt35", agent_state.timezone, tool_call_id)else:- initial_boot_messages = get_initial_boot_messages("startup_with_send_message", agent_state.timezone)- messages = (- [- {"role": "system", "content": full_system_message},- ]- + initial_boot_messages- + [- {"role": "user", "content": first_user_message},- ]- )+ initial_boot_messages = get_initial_boot_messages("startup_with_send_message", agent_state.timezone, tool_call_id)++ # Some LMStudio models (e.g. meta-llama-3.1) require the user message before any tool calls+ if llm_config.provider_name == "lmstudio_openai":+ messages = (+ [+ {"role": "system", "content": full_system_message},+ ]+ + [+ {"role": "user", "content": first_user_message},+ ]+ + initial_boot_messages+ )+ else:+ messages = (+ [+ {"role": "system", "content": full_system_message},+ ]+ + initial_boot_messages+ + [+ {"role": "user", "content": first_user_message},+ ]+ )else:messages = [diff --git a/letta/system.py b/letta/system.pyindex f64cc9d0b..a44f5f7b6 100644--- a/letta/system.py+++ b/letta/system.py@@ -1,5 +1,4 @@import json-import uuidimport warningsfrom typing import Optional@@ -13,7 +12,7 @@ from .helpers.datetime_helpers import get_local_timefrom .helpers.json_helpers import json_dumps-def get_initial_boot_messages(version, timezone):+def get_initial_boot_messages(version, timezone, tool_call_id):if version == "startup":initial_boot_message = INITIAL_BOOT_MESSAGEmessages = [@@ -21,7 +20,6 @@ def get_initial_boot_messages(version, timezone):]elif version == "startup_with_send_message":- tool_call_id = str(uuid.uuid4())messages = [# first message includes both inner monologue and function call to send_message{@@ -53,7 +51,6 @@ def get_initial_boot_messages(version, timezone):]elif version == "startup_with_send_message_gpt35":- tool_call_id = str(uuid.uuid4())messages = [# first message includes both inner monologue and function call to send_message{