Skip to content

armgit / letta

Public Imported from GitHub
0

Platform for stateful agents: AI with advanced memory that can learn and self-improve over time.

feat: Fix VLLM tool_choice (#1001)

Matthew Zhou7dc98969a825
diff --git a/letta/llm_api/llm_api_tools.py b/letta/llm_api/llm_api_tools.py
index 65bdc1f12..e4bc63f63 100644
--- a/letta/llm_api/llm_api_tools.py
+++ b/letta/llm_api/llm_api_tools.py
@@ -151,7 +151,8 @@ def create(
if function_call is None and functions is not None and len(functions) > 0:
# force function calling for reliability, see https://platform.openai.com/docs/api-reference/chat/create#chat-create-tool_choice
# TODO(matt) move into LLMConfig
- if llm_config.model_endpoint == "https://inference.memgpt.ai":
+ # TODO: This vllm checking is very brittle and is a patch at most
+ if llm_config.model_endpoint == "https://inference.memgpt.ai" or (llm_config.handle and "vllm" in llm_config.handle):
function_call = "auto" # TODO change to "required" once proxy supports it
else:
function_call = "required"