Skip to content

armgit / letta

Public Imported from GitHub
0

Platform for stateful agents: AI with advanced memory that can learn and self-improve over time.

refactor(sglang): use input_ids instead of text in native adapter (#9649)

Kevin Lin3eb7fb0a31e6
diff --git a/fern/openapi.json b/fern/openapi.json
index 2dde1ada6..298b748f0 100644
--- a/fern/openapi.json
+++ b/fern/openapi.json
@@ -27752,6 +27752,9 @@
{
"$ref": "#/components/schemas/OpenAIModelSettings"
},
+ {
+ "$ref": "#/components/schemas/SGLangModelSettings"
+ },
{
"$ref": "#/components/schemas/AnthropicModelSettings"
},
@@ -27802,6 +27805,7 @@
"groq": "#/components/schemas/GroqModelSettings",
"openai": "#/components/schemas/OpenAIModelSettings",
"openrouter": "#/components/schemas/OpenRouterModelSettings",
+ "sglang": "#/components/schemas/SGLangModelSettings",
"together": "#/components/schemas/TogetherModelSettings",
"xai": "#/components/schemas/XAIModelSettings",
"zai": "#/components/schemas/ZAIModelSettings"
@@ -31728,6 +31732,9 @@
{
"$ref": "#/components/schemas/OpenAIModelSettings"
},
+ {
+ "$ref": "#/components/schemas/SGLangModelSettings"
+ },
{
"$ref": "#/components/schemas/AnthropicModelSettings"
},
@@ -31778,6 +31785,7 @@
"groq": "#/components/schemas/GroqModelSettings",
"openai": "#/components/schemas/OpenAIModelSettings",
"openrouter": "#/components/schemas/OpenRouterModelSettings",
+ "sglang": "#/components/schemas/SGLangModelSettings",
"together": "#/components/schemas/TogetherModelSettings",
"xai": "#/components/schemas/XAIModelSettings",
"zai": "#/components/schemas/ZAIModelSettings"
@@ -31865,6 +31873,9 @@
{
"$ref": "#/components/schemas/OpenAIModelSettings"
},
+ {
+ "$ref": "#/components/schemas/SGLangModelSettings"
+ },
{
"$ref": "#/components/schemas/AnthropicModelSettings"
},
@@ -31915,6 +31926,7 @@
"groq": "#/components/schemas/GroqModelSettings",
"openai": "#/components/schemas/OpenAIModelSettings",
"openrouter": "#/components/schemas/OpenRouterModelSettings",
+ "sglang": "#/components/schemas/SGLangModelSettings",
"together": "#/components/schemas/TogetherModelSettings",
"xai": "#/components/schemas/XAIModelSettings",
"zai": "#/components/schemas/ZAIModelSettings"
@@ -32563,6 +32575,9 @@
{
"$ref": "#/components/schemas/OpenAIModelSettings"
},
+ {
+ "$ref": "#/components/schemas/SGLangModelSettings"
+ },
{
"$ref": "#/components/schemas/AnthropicModelSettings"
},
@@ -32613,6 +32628,7 @@
"groq": "#/components/schemas/GroqModelSettings",
"openai": "#/components/schemas/OpenAIModelSettings",
"openrouter": "#/components/schemas/OpenRouterModelSettings",
+ "sglang": "#/components/schemas/SGLangModelSettings",
"together": "#/components/schemas/TogetherModelSettings",
"xai": "#/components/schemas/XAIModelSettings",
"zai": "#/components/schemas/ZAIModelSettings"
@@ -33284,6 +33300,9 @@
{
"$ref": "#/components/schemas/OpenAIModelSettings"
},
+ {
+ "$ref": "#/components/schemas/SGLangModelSettings"
+ },
{
"$ref": "#/components/schemas/AnthropicModelSettings"
},
@@ -33334,6 +33353,7 @@
"groq": "#/components/schemas/GroqModelSettings",
"openai": "#/components/schemas/OpenAIModelSettings",
"openrouter": "#/components/schemas/OpenRouterModelSettings",
+ "sglang": "#/components/schemas/SGLangModelSettings",
"together": "#/components/schemas/TogetherModelSettings",
"xai": "#/components/schemas/XAIModelSettings",
"zai": "#/components/schemas/ZAIModelSettings"
@@ -33983,6 +34003,9 @@
{
"$ref": "#/components/schemas/OpenAIModelSettings"
},
+ {
+ "$ref": "#/components/schemas/SGLangModelSettings"
+ },
{
"$ref": "#/components/schemas/AnthropicModelSettings"
},
@@ -34033,6 +34056,7 @@
"groq": "#/components/schemas/GroqModelSettings",
"openai": "#/components/schemas/OpenAIModelSettings",
"openrouter": "#/components/schemas/OpenRouterModelSettings",
+ "sglang": "#/components/schemas/SGLangModelSettings",
"together": "#/components/schemas/TogetherModelSettings",
"xai": "#/components/schemas/XAIModelSettings",
"zai": "#/components/schemas/ZAIModelSettings"
@@ -37909,6 +37933,9 @@
{
"$ref": "#/components/schemas/OpenAIModelSettings"
},
+ {
+ "$ref": "#/components/schemas/SGLangModelSettings"
+ },
{
"$ref": "#/components/schemas/AnthropicModelSettings"
},
@@ -37959,6 +37986,7 @@
"groq": "#/components/schemas/GroqModelSettings",
"openai": "#/components/schemas/OpenAIModelSettings",
"openrouter": "#/components/schemas/OpenRouterModelSettings",
+ "sglang": "#/components/schemas/SGLangModelSettings",
"together": "#/components/schemas/TogetherModelSettings",
"xai": "#/components/schemas/XAIModelSettings",
"zai": "#/components/schemas/ZAIModelSettings"
@@ -39154,6 +39182,18 @@
"title": "Return Token Ids",
"description": "Whether to return token IDs for all LLM generations via SGLang native endpoint. Required for multi-turn RL training with loss masking. Only works with SGLang provider.",
"default": false
+ },
+ "tool_call_parser": {
+ "anyOf": [
+ {
+ "type": "string"
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "title": "Tool Call Parser",
+ "description": "SGLang tool call parser name (e.g. 'glm47', 'qwen25', 'hermes'). Used by the SGLang native adapter to parse tool calls from raw model output."
}
},
"type": "object",
@@ -41909,6 +41949,18 @@
"description": "Whether to return token IDs for all LLM generations via SGLang native endpoint. Required for multi-turn RL training with loss masking. Only works with SGLang provider.",
"default": false
},
+ "tool_call_parser": {
+ "anyOf": [
+ {
+ "type": "string"
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "title": "Tool Call Parser",
+ "description": "SGLang tool call parser name (e.g. 'glm47', 'qwen25', 'hermes'). Used by the SGLang native adapter to parse tool calls from raw model output."
+ },
"max_context_window": {
"type": "integer",
"title": "Max Context Window",
@@ -44168,6 +44220,93 @@
"title": "RunStatus",
"description": "Status of the run."
},
+ "SGLangModelSettings": {
+ "properties": {
+ "max_output_tokens": {
+ "type": "integer",
+ "title": "Max Output Tokens",
+ "description": "The maximum number of tokens the model can generate.",
+ "default": 4096
+ },
+ "parallel_tool_calls": {
+ "type": "boolean",
+ "title": "Parallel Tool Calls",
+ "description": "Whether to enable parallel tool calling.",
+ "default": true
+ },
+ "provider_type": {
+ "type": "string",
+ "const": "sglang",
+ "title": "Provider Type",
+ "description": "The type of the provider.",
+ "default": "sglang"
+ },
+ "temperature": {
+ "type": "number",
+ "title": "Temperature",
+ "description": "The temperature of the model.",
+ "default": 0.7
+ },
+ "reasoning": {
+ "$ref": "#/components/schemas/OpenAIReasoning",
+ "description": "The reasoning configuration for the model.",
+ "default": {
+ "reasoning_effort": "high"
+ }
+ },
+ "response_format": {
+ "anyOf": [
+ {
+ "oneOf": [
+ {
+ "$ref": "#/components/schemas/TextResponseFormat"
+ },
+ {
+ "$ref": "#/components/schemas/JsonSchemaResponseFormat"
+ },
+ {
+ "$ref": "#/components/schemas/JsonObjectResponseFormat"
+ }
+ ],
+ "discriminator": {
+ "propertyName": "type",
+ "mapping": {
+ "json_object": "#/components/schemas/JsonObjectResponseFormat",
+ "json_schema": "#/components/schemas/JsonSchemaResponseFormat",
+ "text": "#/components/schemas/TextResponseFormat"
+ }
+ }
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "title": "Response Format",
+ "description": "The response format for the model."
+ },
+ "strict": {
+ "type": "boolean",
+ "title": "Strict",
+ "description": "Enable strict mode for tool calling. When true, tool outputs are guaranteed to match JSON schemas.",
+ "default": true
+ },
+ "tool_call_parser": {
+ "anyOf": [
+ {
+ "type": "string"
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "title": "Tool Call Parser",
+ "description": "SGLang tool call parser name (for example 'glm47', 'qwen25', or 'hermes')."
+ }
+ },
+ "type": "object",
+ "title": "SGLangModelSettings",
+ "description": "SGLang model configuration (OpenAI-compatible runtime with SGLang-specific parsing)."
+ },
"SSEMCPServer": {
"properties": {
"mcp_server_type": {
@@ -48405,6 +48544,9 @@
{
"$ref": "#/components/schemas/OpenAIModelSettings"
},
+ {
+ "$ref": "#/components/schemas/SGLangModelSettings"
+ },
{
"$ref": "#/components/schemas/AnthropicModelSettings"
},
@@ -48455,6 +48597,7 @@
"groq": "#/components/schemas/GroqModelSettings",
"openai": "#/components/schemas/OpenAIModelSettings",
"openrouter": "#/components/schemas/OpenRouterModelSettings",
+ "sglang": "#/components/schemas/SGLangModelSettings",
"together": "#/components/schemas/TogetherModelSettings",
"xai": "#/components/schemas/XAIModelSettings",
"zai": "#/components/schemas/ZAIModelSettings"
@@ -48745,6 +48888,9 @@
{
"$ref": "#/components/schemas/OpenAIModelSettings"
},
+ {
+ "$ref": "#/components/schemas/SGLangModelSettings"
+ },
{
"$ref": "#/components/schemas/AnthropicModelSettings"
},
@@ -48795,6 +48941,7 @@
"groq": "#/components/schemas/GroqModelSettings",
"openai": "#/components/schemas/OpenAIModelSettings",
"openrouter": "#/components/schemas/OpenRouterModelSettings",
+ "sglang": "#/components/schemas/SGLangModelSettings",
"together": "#/components/schemas/TogetherModelSettings",
"xai": "#/components/schemas/XAIModelSettings",
"zai": "#/components/schemas/ZAIModelSettings"
@@ -49953,6 +50100,9 @@
{
"$ref": "#/components/schemas/OpenAIModelSettings"
},
+ {
+ "$ref": "#/components/schemas/SGLangModelSettings"
+ },
{
"$ref": "#/components/schemas/AnthropicModelSettings"
},
@@ -50003,6 +50153,7 @@
"groq": "#/components/schemas/GroqModelSettings",
"openai": "#/components/schemas/OpenAIModelSettings",
"openrouter": "#/components/schemas/OpenRouterModelSettings",
+ "sglang": "#/components/schemas/SGLangModelSettings",
"together": "#/components/schemas/TogetherModelSettings",
"xai": "#/components/schemas/XAIModelSettings",
"zai": "#/components/schemas/ZAIModelSettings"
diff --git a/letta/adapters/sglang_native_adapter.py b/letta/adapters/sglang_native_adapter.py
index cab8b267d..01ce92558 100644
--- a/letta/adapters/sglang_native_adapter.py
+++ b/letta/adapters/sglang_native_adapter.py
@@ -1,11 +1,8 @@
"""
SGLang Native Adapter for multi-turn RL training.
-This adapter uses SGLang's native /generate endpoint instead of the OpenAI-compatible
-endpoint to get token IDs and per-token logprobs, which are essential for proper
-multi-turn RL training with loss masking.
-
-Uses HuggingFace tokenizer's apply_chat_template() for proper tool formatting.
+Uses SGLang's native /generate endpoint with input_ids (pre-tokenized via HF
+apply_chat_template) to get token IDs and per-token logprobs for loss masking.
"""
import json
@@ -18,8 +15,10 @@ from letta.adapters.simple_llm_request_adapter import SimpleLLMRequestAdapter
from letta.helpers.datetime_helpers import get_utc_timestamp_ns
from letta.llm_api.sglang_native_client import SGLangNativeClient
from letta.log import get_logger
+from letta.schemas.enums import ProviderType
from letta.schemas.letta_message import LettaMessage
from letta.schemas.letta_message_content import TextContent
+from letta.schemas.model import ModelSettingsUnion
from letta.schemas.openai.chat_completion_response import (
ChatCompletionResponse,
ChatCompletionTokenLogprob,
@@ -33,187 +32,141 @@ from letta.schemas.openai.chat_completion_response import (
logger = get_logger(__name__)
-# Global tokenizer cache
+# Global tokenizer cache keyed by model name
_tokenizer_cache: dict[str, Any] = {}
-class SGLangNativeAdapter(SimpleLLMRequestAdapter):
- """
- Adapter that uses SGLang's native /generate endpoint for multi-turn RL training.
+def _get_tokenizer(model_name: str) -> Any:
+ """Load and cache HF tokenizer for the given model."""
+ if model_name in _tokenizer_cache:
+ return _tokenizer_cache[model_name]
- Key differences from SimpleLLMRequestAdapter:
- - Uses /generate instead of /v1/chat/completions
- - Returns output_ids (token IDs) in addition to text
- - Returns output_token_logprobs with [logprob, token_id] pairs
- - Formats tools into prompt and parses tool calls from response
+ from transformers import AutoTokenizer
- These are essential for building accurate loss masks in multi-turn training.
- """
+ logger.info(f"Loading tokenizer for {model_name}")
+ tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
+ _tokenizer_cache[model_name] = tokenizer
+ return tokenizer
- def __init__(self, *args, **kwargs):
- super().__init__(*args, **kwargs)
- self._sglang_client: Optional[SGLangNativeClient] = None
- self._tokenizer: Any = None
-
- def _get_tokenizer(self) -> Any:
- """Get or create tokenizer for the model."""
- global _tokenizer_cache
- # Get model name from llm_config
- model_name = self.llm_config.model
- if not model_name:
- logger.warning("No model name in llm_config, cannot load tokenizer")
- return None
+def _resolve_tokenizer_path(model_name: str) -> str:
+ """Resolve model name to a tokenizer-loadable path.
- # Check cache
- if model_name in _tokenizer_cache:
- return _tokenizer_cache[model_name]
+ Handles handles like 'sglang/slime-sglang//local/path' or
+ 'slime-sglang//local/path' by extracting the local filesystem path.
+ """
+ # Strip leading provider prefixes (e.g. 'sglang/', 'openai-proxy/')
+ # until we find either a HF repo id or a local path
+ parts = model_name.split("/")
+ # Reconstruct: find where the absolute path starts (leading '/')
+ # e.g. "slime-sglang//opt/..." -> after splitting on '/' gives ['slime-sglang', '', 'opt', ...]
+ # join from first empty string onward to recover '/opt/...'
+ for i, part in enumerate(parts):
+ if part == "" and i > 0:
+ local_path = "/" + "/".join(parts[i + 1 :])
+ if local_path != "/":
+ return local_path
+ return model_name
+
+
+def _messages_to_input_ids(model_name: str, messages: list, tools: list) -> list[int]:
+ """Apply the model's chat template and return token IDs.
+
+ Uses apply_chat_template(tokenize=True) — single step, correct template,
+ no double tokenization. Raises clearly if tokenizer cannot be loaded.
+ """
+ tokenizer = _get_tokenizer(_resolve_tokenizer_path(model_name))
+
+ openai_messages = _to_openai_messages(messages)
+ openai_tools = _to_openai_tools(tools) if tools else None
+
+ result = tokenizer.apply_chat_template(
+ openai_messages,
+ tokenize=True,
+ add_generation_prompt=True,
+ tools=openai_tools,
+ return_tensors=None, # plain Python list
+ )
+ # apply_chat_template may return a BatchEncoding (dict-like) or a plain list.
+ # BatchEncoding doesn't pass isinstance(dict) checks, so use hasattr instead.
+ if hasattr(result, "input_ids"):
+ return list(result.input_ids)
+ if hasattr(result, "__getitem__") and not isinstance(result, (list, tuple)):
+ return list(result["input_ids"])
+ return list(result)
+
+
+def _to_openai_messages(messages: list) -> list[dict]:
+ """Convert Letta Message objects to OpenAI-style dicts."""
+ result = []
+ for msg in messages:
+ if hasattr(msg, "role"):
+ role = msg.role
+ content = msg.content or ""
+ if isinstance(content, list):
+ parts = []
+ for c in content:
+ text = getattr(c, "text", None)
+ if text is not None:
+ parts.append(text)
+ content = "\n".join(parts) if parts else ""
+ tool_calls = getattr(msg, "tool_calls", None)
+ tool_call_id = getattr(msg, "tool_call_id", None)
+ name = getattr(msg, "name", None)
+ else:
+ role = msg.get("role", "user")
+ content = msg.get("content", "") or ""
+ tool_calls = msg.get("tool_calls")
+ tool_call_id = msg.get("tool_call_id")
+ name = msg.get("name")
+
+ d: dict = {"role": role, "content": content}
+
+ if tool_calls:
+ parsed_calls = []
+ for tc in tool_calls:
+ if hasattr(tc, "function"):
+ tc_id = getattr(tc, "id", None) or f"call_{uuid.uuid4().hex[:8]}"
+ tc_name = tc.function.name
+ tc_args = tc.function.arguments
+ else:
+ tc_id = tc.get("id", f"call_{uuid.uuid4().hex[:8]}")
+ tc_name = tc.get("function", {}).get("name", "")
+ tc_args = tc.get("function", {}).get("arguments", "{}")
+ # GLM chat template expects arguments as dict, not JSON string
+ if isinstance(tc_args, str):
+ try:
+ tc_args = json.loads(tc_args)
+ except (json.JSONDecodeError, ValueError):
+ tc_args = {}
+ parsed_calls.append(
+ {
+ "id": tc_id,
+ "type": "function",
+ "function": {"name": tc_name, "arguments": tc_args},
+ }
+ )
+ d["tool_calls"] = parsed_calls
- try:
- from transformers import AutoTokenizer
-
- logger.info(f"Loading tokenizer for model: {model_name}")
- tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
- _tokenizer_cache[model_name] = tokenizer
- return tokenizer
- except ImportError:
- logger.warning("transformers not installed, falling back to manual formatting")
- return None
- except Exception as e:
- logger.warning(f"Failed to load tokenizer: {e}, falling back to manual formatting")
- return None
+ if tool_call_id:
+ d["tool_call_id"] = tool_call_id
+ if name and role == "tool":
+ d["name"] = name
- def _get_sglang_client(self) -> SGLangNativeClient:
- """Get or create SGLang native client."""
- if self._sglang_client is None:
- # Get base URL from llm_config, removing /v1 suffix if present
- base_url = self.llm_config.model_endpoint or ""
- # SGLang local instances typically don't need API key
- self._sglang_client = SGLangNativeClient(
- base_url=base_url,
- api_key=None,
- )
- return self._sglang_client
+ result.append(d)
+ return result
- def _format_tools_for_prompt(self, tools: list) -> str:
- """
- Format tools in Qwen3 chat template format for the system prompt.
-
- This matches the exact format produced by Qwen3's tokenizer.apply_chat_template()
- with tools parameter.
- """
- if not tools:
- return ""
-
- # Format each tool as JSON (matching Qwen3 template exactly)
- tool_jsons = []
- for tool in tools:
- # Handle both dict and object formats
- if isinstance(tool, dict):
- # Already in OpenAI format
- tool_jsons.append(json.dumps(tool))
- else:
- # Convert object to dict
- tool_dict = {
- "type": "function",
- "function": {
- "name": getattr(getattr(tool, "function", tool), "name", ""),
- "description": getattr(getattr(tool, "function", tool), "description", ""),
- "parameters": getattr(getattr(tool, "function", tool), "parameters", {}),
- },
- }
- tool_jsons.append(json.dumps(tool_dict))
-
- # Use exact Qwen3 format
- tools_section = (
- "\n\n# Tools\n\n"
- "You may call one or more functions to assist with the user query.\n\n"
- "You are provided with function signatures within <tools></tools> XML tags:\n"
- "<tools>\n" + "\n".join(tool_jsons) + "\n"
- "</tools>\n\n"
- "For each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n"
- "<tool_call>\n"
- '{"name": <function-name>, "arguments": <args-json-object>}\n'
- "</tool_call>"
- )
- return tools_section
-
- def _convert_messages_to_openai_format(self, messages: list) -> list[dict]:
- """Convert Letta Message objects to OpenAI-style message dicts."""
- openai_messages = []
-
- for msg in messages:
- # Handle both dict and Pydantic Message objects
- if hasattr(msg, "role"):
- role = msg.role
- content = msg.content if hasattr(msg, "content") else ""
- # Handle content that might be a list of content parts
- if isinstance(content, list):
- content = " ".join([c.text if hasattr(c, "text") else str(c) for c in content])
- elif content is None:
- content = ""
- tool_calls = getattr(msg, "tool_calls", None)
- tool_call_id = getattr(msg, "tool_call_id", None)
- name = getattr(msg, "name", None)
- else:
- role = msg.get("role", "user")
- content = msg.get("content", "")
- tool_calls = msg.get("tool_calls", None)
- tool_call_id = msg.get("tool_call_id", None)
- name = msg.get("name", None)
-
- openai_msg = {"role": role, "content": content}
-
- if tool_calls:
- # Convert tool calls to OpenAI format
- openai_tool_calls = []
- for tc in tool_calls:
- if hasattr(tc, "function"):
- tc_dict = {
- "id": getattr(tc, "id", f"call_{uuid.uuid4().hex[:8]}"),
- "type": "function",
- "function": {
- "name": tc.function.name,
- "arguments": tc.function.arguments
- if isinstance(tc.function.arguments, str)
- else json.dumps(tc.function.arguments),
- },
- }
- else:
- tc_dict = {
- "id": tc.get("id", f"call_{uuid.uuid4().hex[:8]}"),
- "type": "function",
- "function": tc.get("function", {}),
- }
- openai_tool_calls.append(tc_dict)
- openai_msg["tool_calls"] = openai_tool_calls
-
- if tool_call_id:
- openai_msg["tool_call_id"] = tool_call_id
-
- if name and role == "tool":
- openai_msg["name"] = name
-
- openai_messages.append(openai_msg)
-
- return openai_messages
-
- def _convert_tools_to_openai_format(self, tools: list) -> list[dict]:
- """Convert tools to OpenAI format for tokenizer."""
- openai_tools = []
- for tool in tools:
- if isinstance(tool, dict):
- # Already a dict, ensure it's in the right format
- if "function" in tool:
- openai_tools.append(tool)
- else:
- # Might be the function directly
- openai_tools.append({"type": "function", "function": tool})
- else:
- # Convert object to dict
- func = getattr(tool, "function", tool)
- tool_dict = {
+def _to_openai_tools(tools: list) -> list[dict]:
+ """Convert tool objects to OpenAI-format dicts."""
+ result = []
+ for tool in tools:
+ if isinstance(tool, dict):
+ result.append(tool if "function" in tool else {"type": "function", "function": tool})
+ else:
+ func = getattr(tool, "function", tool)
+ result.append(
+ {
"type": "function",
"function": {
"name": getattr(func, "name", ""),
@@ -221,138 +174,117 @@ class SGLangNativeAdapter(SimpleLLMRequestAdapter):
"parameters": getattr(func, "parameters", {}),
},
}
- openai_tools.append(tool_dict)
- return openai_tools
-
- def _format_messages_to_text(self, messages: list, tools: list) -> str:
- """
- Format messages to text using tokenizer's apply_chat_template if available.
+ )
+ return result
- Falls back to manual formatting if tokenizer is not available.
- """
- tokenizer = self._get_tokenizer()
- if tokenizer is not None:
- # Use tokenizer's apply_chat_template for proper formatting
- openai_messages = self._convert_messages_to_openai_format(messages)
- openai_tools = self._convert_tools_to_openai_format(tools) if tools else None
+def _parse_glm47_tool_calls(text: str) -> list[ToolCall]:
+ """Parse GLM-4.7 XML tool call format inline (no sglang dependency).
+ Format: <tool_call>func_name<arg_key>k</arg_key><arg_value>v</arg_value>...</tool_call>
+ """
+ tool_calls = []
+ for inner in re.findall(r"<tool_call>(.*?)</tool_call>", text, re.DOTALL):
+ inner = inner.strip()
+ # JSON format fallback
+ if inner.startswith("{"):
try:
- formatted = tokenizer.apply_chat_template(
- openai_messages,
- tokenize=False,
- add_generation_prompt=True,
- tools=openai_tools,
+ data = json.loads(inner)
+ args = data.get("arguments", {})
+ if isinstance(args, dict):
+ args = json.dumps(args)
+ tool_calls.append(
+ ToolCall(
+ id=f"call_{uuid.uuid4().hex[:8]}",
+ type="function",
+ function=FunctionCall(name=data.get("name", ""), arguments=args),
+ )
)
- logger.debug(f"Formatted prompt using tokenizer ({len(formatted)} chars)")
- return formatted
- except Exception as e:
- logger.warning(f"apply_chat_template failed: {e}, falling back to manual formatting")
-
- # Fallback to manual formatting
- return self._format_messages_to_text_manual(messages, tools)
-
- def _format_messages_to_text_manual(self, messages: list, tools: list) -> str:
- """Manual fallback formatting for when tokenizer is not available."""
- formatted_parts = []
- tools_section = self._format_tools_for_prompt(tools)
-
- for msg in messages:
- # Handle both dict and Pydantic Message objects
- if hasattr(msg, "role"):
- role = msg.role
- content = msg.content if hasattr(msg, "content") else ""
- if isinstance(content, list):
- content = " ".join([c.text if hasattr(c, "text") else str(c) for c in content])
- elif content is None:
- content = ""
- tool_calls = getattr(msg, "tool_calls", None)
- else:
- role = msg.get("role", "user")
- content = msg.get("content", "")
- tool_calls = msg.get("tool_calls", None)
-
- if role == "system":
- system_content = content + tools_section if tools_section else content
- formatted_parts.append(f"<|im_start|>system\n{system_content}<|im_end|>")
- tools_section = ""
- elif role == "user":
- formatted_parts.append(f"<|im_start|>user\n{content}<|im_end|>")
- elif role == "assistant":
- if tool_calls:
- tc_parts = []
- for tc in tool_calls:
- if hasattr(tc, "function"):
- tc_name = tc.function.name
- tc_args = tc.function.arguments
- else:
- tc_name = tc.get("function", {}).get("name", "")
- tc_args = tc.get("function", {}).get("arguments", "{}")
-
- if isinstance(tc_args, str):
- try:
- tc_args = json.loads(tc_args)
- except Exception:
- pass
-
- tc_parts.append(f'<tool_call>\n{{"name": "{tc_name}", "arguments": {json.dumps(tc_args)}}}\n</tool_call>')
-
- assistant_content = content + "\n" + "\n".join(tc_parts) if content else "\n".join(tc_parts)
- formatted_parts.append(f"<|im_start|>assistant\n{assistant_content}<|im_end|>")
- elif content:
- formatted_parts.append(f"<|im_start|>assistant\n{content}<|im_end|>")
- elif role == "tool":
- formatted_parts.append(f"<|im_start|>user\n<tool_response>\n{content}\n</tool_response><|im_end|>")
-
- formatted_parts.append("<|im_start|>assistant\n")
- return "\n".join(formatted_parts)
-
- def _parse_tool_calls(self, text: str) -> list[ToolCall]:
- """
- Parse tool calls from response text.
-
- Looks for patterns like:
- <tool_call>
- {"name": "tool_name", "arguments": {...}}
- </tool_call>
- """
- tool_calls = []
-
- # Find all tool_call blocks
- pattern = r"<tool_call>\s*(\{.*?\})\s*</tool_call>"
- matches = re.findall(pattern, text, re.DOTALL)
+ continue
+ except json.JSONDecodeError:
+ pass
+ # GLM-4.7 XML format: first non-tag text is function name
+ name_match = re.match(r"^([^<\n]+)", inner)
+ if not name_match:
+ continue
+ func_name = name_match.group(1).strip()
+ keys = re.findall(r"<arg_key>(.*?)</arg_key>", inner, re.DOTALL)
+ vals = re.findall(r"<arg_value>(.*?)</arg_value>", inner, re.DOTALL)
+ tool_calls.append(
+ ToolCall(
+ id=f"call_{uuid.uuid4().hex[:8]}",
+ type="function",
+ function=FunctionCall(name=func_name, arguments=json.dumps(dict(zip(keys, vals)))),
+ )
+ )
+ return tool_calls
- for match in matches:
- try:
- tc_data = json.loads(match)
- name = tc_data.get("name", "")
- arguments = tc_data.get("arguments", {})
- if isinstance(arguments, dict):
- arguments = json.dumps(arguments)
+def _parse_tool_calls(text: str, tools: list | None = None, tool_call_parser: str = "glm47") -> list[ToolCall]:
+ """Parse tool calls from response text.
- tool_call = ToolCall(
+ Tries SGLang's FunctionCallParser first; falls back to inline GLM-4.7 parser.
+ """
+ try:
+ from sglang.srt.function_call.core_types import Function, Tool
+ from sglang.srt.function_call.function_call_parser import FunctionCallParser
+
+ sglang_tools: list[Tool] = []
+ for t in tools or []:
+ func = t.get("function", t) if isinstance(t, dict) else getattr(t, "function", t)
+ name = func.get("name", "") if isinstance(func, dict) else getattr(func, "name", "")
+ desc = func.get("description", "") if isinstance(func, dict) else getattr(func, "description", "")
+ params = func.get("parameters", {}) if isinstance(func, dict) else getattr(func, "parameters", {})
+ sglang_tools.append(Tool(type="function", function=Function(name=name, description=desc, parameters=params)))
+
+ parser = FunctionCallParser(tools=sglang_tools, tool_call_parser=tool_call_parser)
+ result = parser.detector.detect_and_parse(text, sglang_tools)
+ tool_calls = []
+ for call in result.calls:
+ arguments = call.parameters if isinstance(call.parameters, str) else json.dumps(call.parameters)
+ tool_calls.append(
+ ToolCall(
id=f"call_{uuid.uuid4().hex[:8]}",
type="function",
- function=FunctionCall(
- name=name,
- arguments=arguments,
- ),
+ function=FunctionCall(name=call.name, arguments=arguments),
)
- tool_calls.append(tool_call)
- except json.JSONDecodeError as e:
- logger.warning(f"Failed to parse tool call JSON: {e}")
- continue
-
+ )
return tool_calls
+ except ImportError:
+ pass
+ except Exception as e:
+ logger.warning(f"SGLang tool call parser ({tool_call_parser}) failed: {e}")
+
+ # Inline GLM-4.7 fallback (no sglang required)
+ return _parse_glm47_tool_calls(text)
+
+
+def _strip_tool_calls(text: str) -> str:
+ return re.sub(r"<tool_call>.*?</tool_call>", "", text, flags=re.DOTALL).strip()
+
+
+class SGLangNativeAdapter(SimpleLLMRequestAdapter):
+ """
+ Adapter using SGLang's native /generate endpoint for multi-turn RL training.
- def _extract_content_without_tool_calls(self, text: str) -> str:
- """Extract content from response, removing tool_call blocks."""
- # Remove tool_call blocks
- cleaned = re.sub(r"<tool_call>.*?</tool_call>", "", text, flags=re.DOTALL)
- # Clean up whitespace
- cleaned = cleaned.strip()
- return cleaned
+ Flow:
+ 1. Apply model's chat template via HF tokenizer → input_ids (single tokenization step)
+ 2. POST input_ids to SGLang /generate → output_ids + output_token_logprobs
+ 3. Parse tool calls from response text
+ """
+
+ def __init__(self, *args, model_settings: ModelSettingsUnion | None = None, **kwargs):
+ self.model_settings = model_settings
+ super().__init__(*args, **kwargs)
+ self._sglang_client: Optional[SGLangNativeClient] = None
+
+ def _get_sglang_client(self) -> SGLangNativeClient:
+ if self._sglang_client is None:
+ base_url = (self.llm_config.model_endpoint or "").rstrip("/")
+ if base_url.endswith("/v1"):
+ base_url = base_url[:-3]
+ self._sglang_client = SGLangNativeClient(base_url=base_url, api_key=None)
+ return self._sglang_client
async def invoke_llm(
self,
@@ -364,98 +296,73 @@ class SGLangNativeAdapter(SimpleLLMRequestAdapter):
step_id: str | None = None,
actor: str | None = None,
) -> AsyncGenerator[LettaMessage | None, None]:
- """
- Execute LLM request using SGLang native endpoint.
-
- This method:
- 1. Formats messages and tools to text using chat template
- 2. Calls SGLang native /generate endpoint
- 3. Extracts output_ids and output_token_logprobs
- 4. Parses tool calls from response
- 5. Converts response to standard format
- """
self.request_data = request_data
- # Get sampling params from request_data
sampling_params = {
"temperature": request_data.get("temperature", 0.7),
"max_new_tokens": request_data.get("max_tokens", 4096),
"top_p": request_data.get("top_p", 0.9),
}
- # Format messages to text (includes tools in prompt)
- text_input = self._format_messages_to_text(messages, tools)
+ # Tokenize via HF apply_chat_template — correct template, no double tokenization
+ openai_msgs = _to_openai_messages(messages)
+ logger.info(f"SGLang native input: {len(openai_msgs)} messages, roles={[m['role'] for m in openai_msgs]}")
+ if openai_msgs:
+ first_content = openai_msgs[0].get("content", "")
+ logger.info(f" first msg content[:200]: {str(first_content)[:200]}")
+ input_ids = _messages_to_input_ids(self.llm_config.model, messages, tools)
- # Call SGLang native endpoint
client = self._get_sglang_client()
-
- try:
- response = await client.generate(
- text=text_input,
- sampling_params=sampling_params,
- return_logprob=True,
- )
- except Exception as e:
- logger.error(f"SGLang native endpoint error: {e}")
- raise
+ response = await client.generate(
+ input_ids=input_ids,
+ sampling_params=sampling_params,
+ return_logprob=True,
+ )
self.llm_request_finish_timestamp_ns = get_utc_timestamp_ns()
-
- # Store native response data
self.response_data = response
- # Extract SGLang native data
self.output_ids = response.get("output_ids")
- # output_token_logprobs is inside meta_info
meta_info = response.get("meta_info", {})
self.output_token_logprobs = meta_info.get("output_token_logprobs")
- # Extract text response
text_response = response.get("text", "")
+ finish_reason_raw = response.get("meta_info", {}).get("finish_reason", {})
+ output_ids_raw = response.get("output_ids", [])
+ logger.info(f"SGLang raw response[:300]: {repr(text_response[:300])}")
+ logger.info(f"SGLang finish_reason: {finish_reason_raw}")
+ logger.info(f"SGLang output_ids[:15]: {output_ids_raw[:15]}")
+ # Decode output_ids directly to verify what tokens were generated
+ try:
+ tok = _get_tokenizer(_resolve_tokenizer_path(self.llm_config.model))
+ logger.info(f"SGLang output decoded[:200]: {repr(tok.decode(output_ids_raw[:50]))}")
+ except Exception as _e:
+ pass
- # Remove trailing end token if present
- if text_response.endswith("<|im_end|>"):
- text_response = text_response[:-10]
-
- # Parse tool calls from response
- parsed_tool_calls = self._parse_tool_calls(text_response)
-
- # Extract content (text without tool_call blocks)
- content_text = self._extract_content_without_tool_calls(text_response)
+ tool_call_parser = "qwen25"
+ if self.model_settings is not None and getattr(self.model_settings, "provider_type", None) == ProviderType.sglang:
+ tool_call_parser = getattr(self.model_settings, "tool_call_parser", None) or tool_call_parser
+ parsed_tool_calls = _parse_tool_calls(text_response, tools=tools, tool_call_parser=tool_call_parser)
+ content_text = _strip_tool_calls(text_response)
- # Determine finish reason
- meta_info = response.get("meta_info", {})
finish_reason_info = meta_info.get("finish_reason", {})
- if isinstance(finish_reason_info, dict):
- finish_reason = finish_reason_info.get("type", "stop")
- else:
- finish_reason = "stop"
-
- # If we have tool calls, set finish_reason to tool_calls
+ finish_reason = finish_reason_info.get("type", "stop") if isinstance(finish_reason_info, dict) else "stop"
if parsed_tool_calls:
finish_reason = "tool_calls"
- # Convert to standard ChatCompletionResponse format for compatibility
- # Build logprobs in OpenAI format from SGLang format
logprobs_content = None
if self.output_token_logprobs:
- logprobs_content = []
- for i, lp_data in enumerate(self.output_token_logprobs):
- # SGLang format: [logprob, token_id, top_logprob]
- logprob = lp_data[0] if len(lp_data) > 0 else 0.0
- token_id = lp_data[1] if len(lp_data) > 1 else 0
- logprobs_content.append(
- ChatCompletionTokenLogprob(
- token=str(token_id),
- logprob=logprob,
- bytes=None,
- top_logprobs=[],
- )
+ logprobs_content = [
+ ChatCompletionTokenLogprob(
+ token=str(lp[1]) if len(lp) > 1 else "0",
+ logprob=lp[0] if len(lp) > 0 else 0.0,
+ bytes=None,
+ top_logprobs=[],
)
+ for lp in self.output_token_logprobs
+ ]
choice_logprobs = ChoiceLogprobs(content=logprobs_content) if logprobs_content else None
-
- # Build chat completion response
prompt_tokens = meta_info.get("prompt_tokens", 0)
completion_tokens = len(self.output_ids) if self.output_ids else 0
@@ -468,8 +375,8 @@ class SGLangNativeAdapter(SimpleLLMRequestAdapter):
index=0,
message=ChoiceMessage(
role="assistant",
- content=content_text if content_text else None,
- tool_calls=parsed_tool_calls if parsed_tool_calls else None,
+ content=content_text or None,
+ tool_calls=parsed_tool_calls or None,
),
logprobs=choice_logprobs,
)
@@ -481,23 +388,12 @@ class SGLangNativeAdapter(SimpleLLMRequestAdapter):
),
)
- # Extract content
- if content_text:
- self.content = [TextContent(text=content_text)]
- else:
- self.content = None
-
- # No reasoning content from native endpoint
+ self.content = [TextContent(text=content_text)] if content_text else None
self.reasoning_content = None
-
- # Set tool calls
self.tool_calls = parsed_tool_calls
self.tool_call = parsed_tool_calls[0] if parsed_tool_calls else None
-
- # Set logprobs
self.logprobs = choice_logprobs
- # Extract usage statistics
self.usage.step_count = 1
self.usage.completion_tokens = completion_tokens
self.usage.prompt_tokens = prompt_tokens
@@ -506,7 +402,7 @@ class SGLangNativeAdapter(SimpleLLMRequestAdapter):
self.log_provider_trace(step_id=step_id, actor=actor)
logger.info(
- f"SGLang native response: {len(self.output_ids or [])} tokens, "
+ f"SGLang native: {len(self.output_ids or [])} output tokens, "
f"{len(self.output_token_logprobs or [])} logprobs, "
f"{len(parsed_tool_calls)} tool calls"
)
diff --git a/letta/agents/letta_agent_v3.py b/letta/agents/letta_agent_v3.py
index cbc3b4459..3addbcb6b 100644
--- a/letta/agents/letta_agent_v3.py
+++ b/letta/agents/letta_agent_v3.py
@@ -282,11 +282,13 @@ class LettaAgentV3(LettaAgentV2):
self.in_context_messages = curr_in_context_messages
- # Check if we should use SGLang native adapter for multi-turn RL training
+ # Check if we should use SGLang native adapter for multi-turn RL training.
+ # Matches handles starting with "sglang/" OR providers named like "*sglang*"
+ # (e.g. "slime-sglang" used in training).
+ _handle = self.agent_state.llm_config.handle or ""
+ _provider = (self.agent_state.llm_config.provider_name or "").lower()
use_sglang_native = (
- self.agent_state.llm_config.return_token_ids
- and self.agent_state.llm_config.handle
- and self.agent_state.llm_config.handle.startswith("sglang/")
+ self.agent_state.llm_config.return_token_ids and _handle and (_handle.startswith("sglang/") or "sglang" in _provider)
)
self.return_token_ids = use_sglang_native
@@ -295,6 +297,7 @@ class LettaAgentV3(LettaAgentV2):
llm_adapter = SGLangNativeAdapter(
llm_client=self.llm_client,
llm_config=self.agent_state.llm_config,
+ model_settings=self.agent_state.model_settings,
call_type=LLMCallType.agent_step,
agent_id=self.agent_state.id,
agent_tags=self.agent_state.tags,
@@ -515,6 +518,7 @@ class LettaAgentV3(LettaAgentV2):
llm_adapter = SGLangNativeAdapter(
llm_client=self.llm_client,
llm_config=self.agent_state.llm_config,
+ model_settings=self.agent_state.model_settings,
call_type=LLMCallType.agent_step,
agent_id=self.agent_state.id,
agent_tags=self.agent_state.tags,
diff --git a/letta/llm_api/sglang_native_client.py b/letta/llm_api/sglang_native_client.py
index ade9adc35..3f2d9af63 100644
--- a/letta/llm_api/sglang_native_client.py
+++ b/letta/llm_api/sglang_native_client.py
@@ -1,15 +1,11 @@
"""
SGLang Native Client for Letta.
-This client uses SGLang's native /generate endpoint instead of the OpenAI-compatible
-/v1/chat/completions endpoint. The native endpoint returns token IDs and per-token
-logprobs, which are essential for multi-turn RL training.
-
-The OpenAI-compatible endpoint only returns token strings, not IDs, making it
-impossible to accurately reconstruct the token sequence for training.
+Uses SGLang's native /generate endpoint with input_ids for exact token-level control.
+Returns token IDs and per-token logprobs essential for multi-turn RL training.
"""
-from typing import Any, Dict, Optional
+from typing import Any, Dict, List, Optional
import httpx
@@ -21,70 +17,50 @@ logger = get_logger(__name__)
class SGLangNativeClient:
"""Client for SGLang's native /generate endpoint.
- Unlike the OpenAI-compatible endpoint, this returns:
+ Uses input_ids instead of text to avoid double tokenization and ensure
+ the correct chat template is applied (via HF tokenizer on the caller side).
+
+ Returns:
- output_ids: List of token IDs
- output_token_logprobs: List of [logprob, token_id, top_logprob] tuples
-
- This is essential for RL training where we need exact token IDs, not re-tokenized text.
"""
def __init__(self, base_url: str, api_key: Optional[str] = None):
- """
- Initialize the SGLang native client.
-
- Args:
- base_url: Base URL for SGLang server (e.g., http://localhost:30000)
- api_key: Optional API key for authentication
- """
- # Remove /v1 suffix if present - native endpoint is at root
self.base_url = base_url.rstrip("/")
if self.base_url.endswith("/v1"):
self.base_url = self.base_url[:-3]
self.api_key = api_key
+ @property
+ def _headers(self) -> dict:
+ headers = {"Content-Type": "application/json"}
+ if self.api_key:
+ headers["Authorization"] = f"Bearer {self.api_key}"
+ return headers
+
async def generate(
self,
- text: str,
+ input_ids: List[int],
sampling_params: Optional[Dict[str, Any]] = None,
return_logprob: bool = True,
) -> Dict[str, Any]:
- """
- Call SGLang's native /generate endpoint.
+ """Call SGLang's native /generate endpoint with pre-tokenized input.
Args:
- text: The formatted prompt text (with chat template applied)
- sampling_params: Sampling parameters (temperature, max_new_tokens, etc.)
- return_logprob: Whether to return logprobs (default True for RL training)
+ input_ids: Token IDs from apply_chat_template(tokenize=True)
+ sampling_params: temperature, max_new_tokens, top_p, etc.
+ return_logprob: Whether to return per-token logprobs (default True for RL)
Returns:
- Response dict with:
- - text: Generated text
- - output_ids: List of token IDs
- - output_token_logprobs: List of [logprob, token_id, top_logprob] tuples
- - meta_info: Metadata including finish_reason, prompt_tokens, etc.
-
- Example response:
{
- "text": "Hello! How can I help?",
- "output_ids": [9707, 0, 2585, 646, 358, 1492, 30],
- "output_token_logprobs": [
- [-0.005, 9707, null],
- [0.0, 0, null],
- ...
- ],
- "meta_info": {
- "finish_reason": {"type": "stop", "matched": 151645},
- "prompt_tokens": 42,
- ...
- }
+ "text": "...",
+ "output_ids": [token_id, ...],
+ "output_token_logprobs": [[logprob, token_id, top_logprob], ...],
+ "meta_info": {"finish_reason": ..., "prompt_tokens": ..., ...}
}
"""
- headers = {"Content-Type": "application/json"}
- if self.api_key:
- headers["Authorization"] = f"Bearer {self.api_key}"
-
payload = {
- "text": text,
+ "input_ids": input_ids,
"sampling_params": sampling_params or {},
"return_logprob": return_logprob,
}
@@ -93,7 +69,7 @@ class SGLangNativeClient:
response = await client.post(
f"{self.base_url}/generate",
json=payload,
- headers=headers,
+ headers=self._headers,
)
response.raise_for_status()
return response.json()
diff --git a/letta/schemas/llm_config.py b/letta/schemas/llm_config.py
index 7106ed319..3e74b7b71 100644
--- a/letta/schemas/llm_config.py
+++ b/letta/schemas/llm_config.py
@@ -129,6 +129,11 @@ class LLMConfig(BaseModel):
description="Whether to return token IDs for all LLM generations via SGLang native endpoint. "
"Required for multi-turn RL training with loss masking. Only works with SGLang provider.",
)
+ tool_call_parser: Optional[str] = Field(
+ None,
+ description="SGLang tool call parser name (e.g. 'glm47', 'qwen25', 'hermes'). "
+ "Used by the SGLang native adapter to parse tool calls from raw model output.",
+ )
@model_validator(mode="before")
@classmethod
@@ -367,12 +372,23 @@ class LLMConfig(BaseModel):
OpenAIModelSettings,
OpenAIReasoning,
OpenRouterModelSettings,
+ SGLangModelSettings,
TogetherModelSettings,
XAIModelSettings,
ZAIModelSettings,
)
if self.model_endpoint_type == "openai":
+ handle = self.handle or ""
+ provider_name = (self.provider_name or "").lower()
+ if handle.startswith("sglang/") or "sglang" in provider_name:
+ return SGLangModelSettings(
+ max_output_tokens=self.max_tokens or 4096,
+ temperature=self.temperature,
+ reasoning=OpenAIReasoning(reasoning_effort=self.reasoning_effort or "minimal"),
+ strict=self.strict,
+ tool_call_parser=self.tool_call_parser,
+ )
return OpenAIModelSettings(
max_output_tokens=self.max_tokens or 4096,
temperature=self.temperature,
diff --git a/letta/schemas/model.py b/letta/schemas/model.py
index 621bd4f09..d143b9d3c 100644
--- a/letta/schemas/model.py
+++ b/letta/schemas/model.py
@@ -130,6 +130,7 @@ class Model(LLMConfig, ModelBase):
"""Returns the JSON schema for the ModelSettings class corresponding to this model's provider."""
PROVIDER_SETTINGS_MAP = {
ProviderType.openai: OpenAIModelSettings,
+ ProviderType.sglang: SGLangModelSettings,
ProviderType.anthropic: AnthropicModelSettings,
ProviderType.google_ai: GoogleAIModelSettings,
ProviderType.google_vertex: GoogleVertexModelSettings,
@@ -257,6 +258,21 @@ class OpenAIModelSettings(ModelSettings):
}
+class SGLangModelSettings(OpenAIModelSettings):
+ """SGLang model configuration (OpenAI-compatible runtime with SGLang-specific parsing)."""
+
+ provider_type: Literal[ProviderType.sglang] = Field(ProviderType.sglang, description="The type of the provider.")
+ tool_call_parser: Optional[str] = Field(
+ None,
+ description="SGLang tool call parser name (for example 'glm47', 'qwen25', or 'hermes').",
+ )
+
+ def _to_legacy_config_params(self) -> dict:
+ params = super()._to_legacy_config_params()
+ params["tool_call_parser"] = self.tool_call_parser
+ return params
+
+
# "thinking": {
# "type": "enabled",
# "budget_tokens": 10000
@@ -516,6 +532,7 @@ class ChatGPTOAuthModelSettings(ModelSettings):
ModelSettingsUnion = Annotated[
Union[
OpenAIModelSettings,
+ SGLangModelSettings,
AnthropicModelSettings,
GoogleAIModelSettings,
GoogleVertexModelSettings,
diff --git a/letta/schemas/providers/sglang.py b/letta/schemas/providers/sglang.py
index 48d5d0130..31f33a8f5 100644
--- a/letta/schemas/providers/sglang.py
+++ b/letta/schemas/providers/sglang.py
@@ -44,7 +44,7 @@ class SGLangProvider(Provider):
configs.append(
LLMConfig(
model=model_name,
- model_endpoint_type="openai", # SGLang is OpenAI-compatible
+ model_endpoint_type="openai", # SGLang exposes OpenAI-compatible API
model_endpoint=base_url,
model_wrapper=self.default_prompt_formatter,
context_window=model.get("max_model_len", 32768),
@@ -52,6 +52,7 @@ class SGLangProvider(Provider):
max_tokens=self.get_default_max_output_tokens(model_name),
provider_name=self.name,
provider_category=self.provider_category,
+
)
)
diff --git a/letta/services/agent_manager.py b/letta/services/agent_manager.py
index 364f306da..73af54829 100644
--- a/letta/services/agent_manager.py
+++ b/letta/services/agent_manager.py
@@ -515,7 +515,6 @@ class AgentManager:
# Will set mode-specific default prompt if no prompt is provided
effective_compaction_settings = effective_compaction_settings.set_mode_specific_prompt()
-
new_agent = AgentModel(
name=agent_create.name,
system=derive_system_message(
diff --git a/letta/services/provider_manager.py b/letta/services/provider_manager.py
index 011577b48..b59f3846e 100644
--- a/letta/services/provider_manager.py
+++ b/letta/services/provider_manager.py
@@ -1062,7 +1062,10 @@ class ProviderManager:
else:
model_endpoint = f"https://api.{provider.provider_type.value}.com/v1"
- # Construct the LLMConfig from the model and provider data
+ # Construct the LLMConfig from the model and provider data.
+ # SGLang providers get return_token_ids/return_logprobs=True so the native
+ # adapter is used and token IDs are returned for RL training.
+ is_sglang = provider.provider_type == ProviderType.sglang
llm_config = LLMConfig(
model=model.name,
model_endpoint_type=model.model_endpoint_type,
@@ -1072,6 +1075,8 @@ class ProviderManager:
provider_name=provider.name,
provider_category=provider.provider_category,
max_tokens=max_tokens,
+ return_token_ids=is_sglang,
+ return_logprobs=is_sglang,
)
return llm_config
diff --git a/tests/managers/test_agent_manager.py b/tests/managers/test_agent_manager.py
index 4e60875e5..4456f59e3 100644
--- a/tests/managers/test_agent_manager.py
+++ b/tests/managers/test_agent_manager.py
@@ -1954,6 +1954,7 @@ async def test_agent_state_schema_unchanged(server: SyncServer):
"return_logprobs",
"top_logprobs",
"return_token_ids",
+ "tool_call_parser",
}
actual_llm_config_fields = set(llm_config_fields.keys())
if actual_llm_config_fields != expected_llm_config_fields: