Skip to content

armgit / letta

Public Imported from GitHub
0

Platform for stateful agents: AI with advanced memory that can learn and self-improve over time.

feat: move stop token logic to base client (#9917)

cthomasae5f5df9332b
diff --git a/letta/llm_api/baseten_client.py b/letta/llm_api/baseten_client.py
index c30bfdf47..069873911 100644
--- a/letta/llm_api/baseten_client.py
+++ b/letta/llm_api/baseten_client.py
@@ -81,10 +81,6 @@ class BasetenClient(OpenAIClient):
for field in ("reasoning_content_signature", "redacted_reasoning_content", "omitted_reasoning_content"):
msg.pop(field, None)
- # Special tokens are used as turn delimiters; stop generation before the model
- # starts predicting the next turn (prevents "<|user|>" leaking into responses)
- data["stop"] = ["<|user|>", "<|assistant|>", "<|observation|>"]
-
return data
def _build_client_kwargs(self, llm_config: LLMConfig) -> dict:
diff --git a/letta/llm_api/openai_client.py b/letta/llm_api/openai_client.py
index 1fbd5595d..64f0b3406 100644
--- a/letta/llm_api/openai_client.py
+++ b/letta/llm_api/openai_client.py
@@ -661,6 +661,11 @@ class OpenAIClient(LLMClientBase):
# If set, then in the backend "medium" thinking is turned on
# request_data["reasoning_effort"] = "medium"
+ # GLM-5 will sometimes send turn delimiters; stop generation before the model
+ # starts predicting the next turn (prevents "<|user|>" leaking into responses)
+ if (model or "").lower().endswith("glm-5"):
+ request_data["stop"] = ["<|user|>", "<|assistant|>", "<|observation|>"]
+
# Add OpenRouter reasoning configuration via extra_body
if is_openrouter and llm_config.enable_reasoner:
reasoning_config = {}