Skip to content

armgit / letta

Public Imported from GitHub
0

Platform for stateful agents: AI with advanced memory that can learn and self-improve over time.

fix: vllm completions provider

Andy Li7e84607709b9
diff --git a/letta/server/server.py b/letta/server/server.py
index f127b1dda..83bcd2220 100644
--- a/letta/server/server.py
+++ b/letta/server/server.py
@@ -67,6 +67,7 @@ from letta.schemas.providers import (
OpenAIProvider,
Provider,
TogetherProvider,
+ VLLMProvider,
XAIProvider,
)
from letta.schemas.sandbox_config import LocalSandboxConfig, SandboxConfigCreate
@@ -361,22 +362,17 @@ class SyncServer(Server):
)
if model_settings.vllm_api_base:
# vLLM exposes both a /chat/completions and a /completions endpoint
- self._enabled_providers.append(
- VLLMCompletionsProvider(
- name="vllm",
- base_url=model_settings.vllm_api_base,
- default_prompt_formatter=model_settings.default_prompt_formatter,
- )
- )
# NOTE: to use the /chat/completions endpoint, you need to specify extra flags on vLLM startup
# see: https://docs.vllm.ai/en/stable/features/tool_calling.html
# e.g. "... --enable-auto-tool-choice --tool-call-parser hermes"
self._enabled_providers.append(
- VLLMChatCompletionsProvider(
+ VLLMProvider(
name="vllm",
base_url=model_settings.vllm_api_base,
+ default_prompt_formatter=model_settings.default_prompt_formatter,
)
)
+
if model_settings.aws_access_key_id and model_settings.aws_secret_access_key and model_settings.aws_default_region:
self._enabled_providers.append(
BedrockProvider(