Platform for stateful agents: AI with advanced memory that can learn and self-improve over time.
refactor(sglang): use input_ids instead of text in native adapter (#9649)
Kevin Lin3eb7fb0a31e6
diff --git a/fern/openapi.json b/fern/openapi.jsonindex 2dde1ada6..298b748f0 100644--- a/fern/openapi.json+++ b/fern/openapi.json@@ -27752,6 +27752,9 @@{"$ref": "#/components/schemas/OpenAIModelSettings"},+ {+ "$ref": "#/components/schemas/SGLangModelSettings"+ },{"$ref": "#/components/schemas/AnthropicModelSettings"},@@ -27802,6 +27805,7 @@"groq": "#/components/schemas/GroqModelSettings","openai": "#/components/schemas/OpenAIModelSettings","openrouter": "#/components/schemas/OpenRouterModelSettings",+ "sglang": "#/components/schemas/SGLangModelSettings","together": "#/components/schemas/TogetherModelSettings","xai": "#/components/schemas/XAIModelSettings","zai": "#/components/schemas/ZAIModelSettings"@@ -31728,6 +31732,9 @@{"$ref": "#/components/schemas/OpenAIModelSettings"},+ {+ "$ref": "#/components/schemas/SGLangModelSettings"+ },{"$ref": "#/components/schemas/AnthropicModelSettings"},@@ -31778,6 +31785,7 @@"groq": "#/components/schemas/GroqModelSettings","openai": "#/components/schemas/OpenAIModelSettings","openrouter": "#/components/schemas/OpenRouterModelSettings",+ "sglang": "#/components/schemas/SGLangModelSettings","together": "#/components/schemas/TogetherModelSettings","xai": "#/components/schemas/XAIModelSettings","zai": "#/components/schemas/ZAIModelSettings"@@ -31865,6 +31873,9 @@{"$ref": "#/components/schemas/OpenAIModelSettings"},+ {+ "$ref": "#/components/schemas/SGLangModelSettings"+ },{"$ref": "#/components/schemas/AnthropicModelSettings"},@@ -31915,6 +31926,7 @@"groq": "#/components/schemas/GroqModelSettings","openai": "#/components/schemas/OpenAIModelSettings","openrouter": "#/components/schemas/OpenRouterModelSettings",+ "sglang": "#/components/schemas/SGLangModelSettings","together": "#/components/schemas/TogetherModelSettings","xai": "#/components/schemas/XAIModelSettings","zai": "#/components/schemas/ZAIModelSettings"@@ -32563,6 +32575,9 @@{"$ref": "#/components/schemas/OpenAIModelSettings"},+ {+ "$ref": "#/components/schemas/SGLangModelSettings"+ },{"$ref": "#/components/schemas/AnthropicModelSettings"},@@ -32613,6 +32628,7 @@"groq": "#/components/schemas/GroqModelSettings","openai": "#/components/schemas/OpenAIModelSettings","openrouter": "#/components/schemas/OpenRouterModelSettings",+ "sglang": "#/components/schemas/SGLangModelSettings","together": "#/components/schemas/TogetherModelSettings","xai": "#/components/schemas/XAIModelSettings","zai": "#/components/schemas/ZAIModelSettings"@@ -33284,6 +33300,9 @@{"$ref": "#/components/schemas/OpenAIModelSettings"},+ {+ "$ref": "#/components/schemas/SGLangModelSettings"+ },{"$ref": "#/components/schemas/AnthropicModelSettings"},@@ -33334,6 +33353,7 @@"groq": "#/components/schemas/GroqModelSettings","openai": "#/components/schemas/OpenAIModelSettings","openrouter": "#/components/schemas/OpenRouterModelSettings",+ "sglang": "#/components/schemas/SGLangModelSettings","together": "#/components/schemas/TogetherModelSettings","xai": "#/components/schemas/XAIModelSettings","zai": "#/components/schemas/ZAIModelSettings"@@ -33983,6 +34003,9 @@{"$ref": "#/components/schemas/OpenAIModelSettings"},+ {+ "$ref": "#/components/schemas/SGLangModelSettings"+ },{"$ref": "#/components/schemas/AnthropicModelSettings"},@@ -34033,6 +34056,7 @@"groq": "#/components/schemas/GroqModelSettings","openai": "#/components/schemas/OpenAIModelSettings","openrouter": "#/components/schemas/OpenRouterModelSettings",+ "sglang": "#/components/schemas/SGLangModelSettings","together": "#/components/schemas/TogetherModelSettings","xai": "#/components/schemas/XAIModelSettings","zai": "#/components/schemas/ZAIModelSettings"@@ -37909,6 +37933,9 @@{"$ref": "#/components/schemas/OpenAIModelSettings"},+ {+ "$ref": "#/components/schemas/SGLangModelSettings"+ },{"$ref": "#/components/schemas/AnthropicModelSettings"},@@ -37959,6 +37986,7 @@"groq": "#/components/schemas/GroqModelSettings","openai": "#/components/schemas/OpenAIModelSettings","openrouter": "#/components/schemas/OpenRouterModelSettings",+ "sglang": "#/components/schemas/SGLangModelSettings","together": "#/components/schemas/TogetherModelSettings","xai": "#/components/schemas/XAIModelSettings","zai": "#/components/schemas/ZAIModelSettings"@@ -39154,6 +39182,18 @@"title": "Return Token Ids","description": "Whether to return token IDs for all LLM generations via SGLang native endpoint. Required for multi-turn RL training with loss masking. Only works with SGLang provider.","default": false+ },+ "tool_call_parser": {+ "anyOf": [+ {+ "type": "string"+ },+ {+ "type": "null"+ }+ ],+ "title": "Tool Call Parser",+ "description": "SGLang tool call parser name (e.g. 'glm47', 'qwen25', 'hermes'). Used by the SGLang native adapter to parse tool calls from raw model output."}},"type": "object",@@ -41909,6 +41949,18 @@"description": "Whether to return token IDs for all LLM generations via SGLang native endpoint. Required for multi-turn RL training with loss masking. Only works with SGLang provider.","default": false},+ "tool_call_parser": {+ "anyOf": [+ {+ "type": "string"+ },+ {+ "type": "null"+ }+ ],+ "title": "Tool Call Parser",+ "description": "SGLang tool call parser name (e.g. 'glm47', 'qwen25', 'hermes'). Used by the SGLang native adapter to parse tool calls from raw model output."+ },"max_context_window": {"type": "integer","title": "Max Context Window",@@ -44168,6 +44220,93 @@"title": "RunStatus","description": "Status of the run."},+ "SGLangModelSettings": {+ "properties": {+ "max_output_tokens": {+ "type": "integer",+ "title": "Max Output Tokens",+ "description": "The maximum number of tokens the model can generate.",+ "default": 4096+ },+ "parallel_tool_calls": {+ "type": "boolean",+ "title": "Parallel Tool Calls",+ "description": "Whether to enable parallel tool calling.",+ "default": true+ },+ "provider_type": {+ "type": "string",+ "const": "sglang",+ "title": "Provider Type",+ "description": "The type of the provider.",+ "default": "sglang"+ },+ "temperature": {+ "type": "number",+ "title": "Temperature",+ "description": "The temperature of the model.",+ "default": 0.7+ },+ "reasoning": {+ "$ref": "#/components/schemas/OpenAIReasoning",+ "description": "The reasoning configuration for the model.",+ "default": {+ "reasoning_effort": "high"+ }+ },+ "response_format": {+ "anyOf": [+ {+ "oneOf": [+ {+ "$ref": "#/components/schemas/TextResponseFormat"+ },+ {+ "$ref": "#/components/schemas/JsonSchemaResponseFormat"+ },+ {+ "$ref": "#/components/schemas/JsonObjectResponseFormat"+ }+ ],+ "discriminator": {+ "propertyName": "type",+ "mapping": {+ "json_object": "#/components/schemas/JsonObjectResponseFormat",+ "json_schema": "#/components/schemas/JsonSchemaResponseFormat",+ "text": "#/components/schemas/TextResponseFormat"+ }+ }+ },+ {+ "type": "null"+ }+ ],+ "title": "Response Format",+ "description": "The response format for the model."+ },+ "strict": {+ "type": "boolean",+ "title": "Strict",+ "description": "Enable strict mode for tool calling. When true, tool outputs are guaranteed to match JSON schemas.",+ "default": true+ },+ "tool_call_parser": {+ "anyOf": [+ {+ "type": "string"+ },+ {+ "type": "null"+ }+ ],+ "title": "Tool Call Parser",+ "description": "SGLang tool call parser name (for example 'glm47', 'qwen25', or 'hermes')."+ }+ },+ "type": "object",+ "title": "SGLangModelSettings",+ "description": "SGLang model configuration (OpenAI-compatible runtime with SGLang-specific parsing)."+ },"SSEMCPServer": {"properties": {"mcp_server_type": {@@ -48405,6 +48544,9 @@{"$ref": "#/components/schemas/OpenAIModelSettings"},+ {+ "$ref": "#/components/schemas/SGLangModelSettings"+ },{"$ref": "#/components/schemas/AnthropicModelSettings"},@@ -48455,6 +48597,7 @@"groq": "#/components/schemas/GroqModelSettings","openai": "#/components/schemas/OpenAIModelSettings","openrouter": "#/components/schemas/OpenRouterModelSettings",+ "sglang": "#/components/schemas/SGLangModelSettings","together": "#/components/schemas/TogetherModelSettings","xai": "#/components/schemas/XAIModelSettings","zai": "#/components/schemas/ZAIModelSettings"@@ -48745,6 +48888,9 @@{"$ref": "#/components/schemas/OpenAIModelSettings"},+ {+ "$ref": "#/components/schemas/SGLangModelSettings"+ },{"$ref": "#/components/schemas/AnthropicModelSettings"},@@ -48795,6 +48941,7 @@"groq": "#/components/schemas/GroqModelSettings","openai": "#/components/schemas/OpenAIModelSettings","openrouter": "#/components/schemas/OpenRouterModelSettings",+ "sglang": "#/components/schemas/SGLangModelSettings","together": "#/components/schemas/TogetherModelSettings","xai": "#/components/schemas/XAIModelSettings","zai": "#/components/schemas/ZAIModelSettings"@@ -49953,6 +50100,9 @@{"$ref": "#/components/schemas/OpenAIModelSettings"},+ {+ "$ref": "#/components/schemas/SGLangModelSettings"+ },{"$ref": "#/components/schemas/AnthropicModelSettings"},@@ -50003,6 +50153,7 @@"groq": "#/components/schemas/GroqModelSettings","openai": "#/components/schemas/OpenAIModelSettings","openrouter": "#/components/schemas/OpenRouterModelSettings",+ "sglang": "#/components/schemas/SGLangModelSettings","together": "#/components/schemas/TogetherModelSettings","xai": "#/components/schemas/XAIModelSettings","zai": "#/components/schemas/ZAIModelSettings"diff --git a/letta/adapters/sglang_native_adapter.py b/letta/adapters/sglang_native_adapter.pyindex cab8b267d..01ce92558 100644--- a/letta/adapters/sglang_native_adapter.py+++ b/letta/adapters/sglang_native_adapter.py@@ -1,11 +1,8 @@"""SGLang Native Adapter for multi-turn RL training.-This adapter uses SGLang's native /generate endpoint instead of the OpenAI-compatible-endpoint to get token IDs and per-token logprobs, which are essential for proper-multi-turn RL training with loss masking.--Uses HuggingFace tokenizer's apply_chat_template() for proper tool formatting.+Uses SGLang's native /generate endpoint with input_ids (pre-tokenized via HF+apply_chat_template) to get token IDs and per-token logprobs for loss masking."""import json@@ -18,8 +15,10 @@ from letta.adapters.simple_llm_request_adapter import SimpleLLMRequestAdapterfrom letta.helpers.datetime_helpers import get_utc_timestamp_nsfrom letta.llm_api.sglang_native_client import SGLangNativeClientfrom letta.log import get_logger+from letta.schemas.enums import ProviderTypefrom letta.schemas.letta_message import LettaMessagefrom letta.schemas.letta_message_content import TextContent+from letta.schemas.model import ModelSettingsUnionfrom letta.schemas.openai.chat_completion_response import (ChatCompletionResponse,ChatCompletionTokenLogprob,@@ -33,187 +32,141 @@ from letta.schemas.openai.chat_completion_response import (logger = get_logger(__name__)-# Global tokenizer cache+# Global tokenizer cache keyed by model name_tokenizer_cache: dict[str, Any] = {}-class SGLangNativeAdapter(SimpleLLMRequestAdapter):- """- Adapter that uses SGLang's native /generate endpoint for multi-turn RL training.+def _get_tokenizer(model_name: str) -> Any:+ """Load and cache HF tokenizer for the given model."""+ if model_name in _tokenizer_cache:+ return _tokenizer_cache[model_name]- Key differences from SimpleLLMRequestAdapter:- - Uses /generate instead of /v1/chat/completions- - Returns output_ids (token IDs) in addition to text- - Returns output_token_logprobs with [logprob, token_id] pairs- - Formats tools into prompt and parses tool calls from response+ from transformers import AutoTokenizer- These are essential for building accurate loss masks in multi-turn training.- """+ logger.info(f"Loading tokenizer for {model_name}")+ tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)+ _tokenizer_cache[model_name] = tokenizer+ return tokenizer- def __init__(self, *args, **kwargs):- super().__init__(*args, **kwargs)- self._sglang_client: Optional[SGLangNativeClient] = None- self._tokenizer: Any = None-- def _get_tokenizer(self) -> Any:- """Get or create tokenizer for the model."""- global _tokenizer_cache- # Get model name from llm_config- model_name = self.llm_config.model- if not model_name:- logger.warning("No model name in llm_config, cannot load tokenizer")- return None+def _resolve_tokenizer_path(model_name: str) -> str:+ """Resolve model name to a tokenizer-loadable path.- # Check cache- if model_name in _tokenizer_cache:- return _tokenizer_cache[model_name]+ Handles handles like 'sglang/slime-sglang//local/path' or+ 'slime-sglang//local/path' by extracting the local filesystem path.+ """+ # Strip leading provider prefixes (e.g. 'sglang/', 'openai-proxy/')+ # until we find either a HF repo id or a local path+ parts = model_name.split("/")+ # Reconstruct: find where the absolute path starts (leading '/')+ # e.g. "slime-sglang//opt/..." -> after splitting on '/' gives ['slime-sglang', '', 'opt', ...]+ # join from first empty string onward to recover '/opt/...'+ for i, part in enumerate(parts):+ if part == "" and i > 0:+ local_path = "/" + "/".join(parts[i + 1 :])+ if local_path != "/":+ return local_path+ return model_name+++def _messages_to_input_ids(model_name: str, messages: list, tools: list) -> list[int]:+ """Apply the model's chat template and return token IDs.++ Uses apply_chat_template(tokenize=True) — single step, correct template,+ no double tokenization. Raises clearly if tokenizer cannot be loaded.+ """+ tokenizer = _get_tokenizer(_resolve_tokenizer_path(model_name))++ openai_messages = _to_openai_messages(messages)+ openai_tools = _to_openai_tools(tools) if tools else None++ result = tokenizer.apply_chat_template(+ openai_messages,+ tokenize=True,+ add_generation_prompt=True,+ tools=openai_tools,+ return_tensors=None, # plain Python list+ )+ # apply_chat_template may return a BatchEncoding (dict-like) or a plain list.+ # BatchEncoding doesn't pass isinstance(dict) checks, so use hasattr instead.+ if hasattr(result, "input_ids"):+ return list(result.input_ids)+ if hasattr(result, "__getitem__") and not isinstance(result, (list, tuple)):+ return list(result["input_ids"])+ return list(result)+++def _to_openai_messages(messages: list) -> list[dict]:+ """Convert Letta Message objects to OpenAI-style dicts."""+ result = []+ for msg in messages:+ if hasattr(msg, "role"):+ role = msg.role+ content = msg.content or ""+ if isinstance(content, list):+ parts = []+ for c in content:+ text = getattr(c, "text", None)+ if text is not None:+ parts.append(text)+ content = "\n".join(parts) if parts else ""+ tool_calls = getattr(msg, "tool_calls", None)+ tool_call_id = getattr(msg, "tool_call_id", None)+ name = getattr(msg, "name", None)+ else:+ role = msg.get("role", "user")+ content = msg.get("content", "") or ""+ tool_calls = msg.get("tool_calls")+ tool_call_id = msg.get("tool_call_id")+ name = msg.get("name")++ d: dict = {"role": role, "content": content}++ if tool_calls:+ parsed_calls = []+ for tc in tool_calls:+ if hasattr(tc, "function"):+ tc_id = getattr(tc, "id", None) or f"call_{uuid.uuid4().hex[:8]}"+ tc_name = tc.function.name+ tc_args = tc.function.arguments+ else:+ tc_id = tc.get("id", f"call_{uuid.uuid4().hex[:8]}")+ tc_name = tc.get("function", {}).get("name", "")+ tc_args = tc.get("function", {}).get("arguments", "{}")+ # GLM chat template expects arguments as dict, not JSON string+ if isinstance(tc_args, str):+ try:+ tc_args = json.loads(tc_args)+ except (json.JSONDecodeError, ValueError):+ tc_args = {}+ parsed_calls.append(+ {+ "id": tc_id,+ "type": "function",+ "function": {"name": tc_name, "arguments": tc_args},+ }+ )+ d["tool_calls"] = parsed_calls- try:- from transformers import AutoTokenizer-- logger.info(f"Loading tokenizer for model: {model_name}")- tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)- _tokenizer_cache[model_name] = tokenizer- return tokenizer- except ImportError:- logger.warning("transformers not installed, falling back to manual formatting")- return None- except Exception as e:- logger.warning(f"Failed to load tokenizer: {e}, falling back to manual formatting")- return None+ if tool_call_id:+ d["tool_call_id"] = tool_call_id+ if name and role == "tool":+ d["name"] = name- def _get_sglang_client(self) -> SGLangNativeClient:- """Get or create SGLang native client."""- if self._sglang_client is None:- # Get base URL from llm_config, removing /v1 suffix if present- base_url = self.llm_config.model_endpoint or ""- # SGLang local instances typically don't need API key- self._sglang_client = SGLangNativeClient(- base_url=base_url,- api_key=None,- )- return self._sglang_client+ result.append(d)+ return result- def _format_tools_for_prompt(self, tools: list) -> str:- """- Format tools in Qwen3 chat template format for the system prompt.-- This matches the exact format produced by Qwen3's tokenizer.apply_chat_template()- with tools parameter.- """- if not tools:- return ""-- # Format each tool as JSON (matching Qwen3 template exactly)- tool_jsons = []- for tool in tools:- # Handle both dict and object formats- if isinstance(tool, dict):- # Already in OpenAI format- tool_jsons.append(json.dumps(tool))- else:- # Convert object to dict- tool_dict = {- "type": "function",- "function": {- "name": getattr(getattr(tool, "function", tool), "name", ""),- "description": getattr(getattr(tool, "function", tool), "description", ""),- "parameters": getattr(getattr(tool, "function", tool), "parameters", {}),- },- }- tool_jsons.append(json.dumps(tool_dict))-- # Use exact Qwen3 format- tools_section = (- "\n\n# Tools\n\n"- "You may call one or more functions to assist with the user query.\n\n"- "You are provided with function signatures within <tools></tools> XML tags:\n"- "<tools>\n" + "\n".join(tool_jsons) + "\n"- "</tools>\n\n"- "For each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n"- "<tool_call>\n"- '{"name": <function-name>, "arguments": <args-json-object>}\n'- "</tool_call>"- )- return tools_section-- def _convert_messages_to_openai_format(self, messages: list) -> list[dict]:- """Convert Letta Message objects to OpenAI-style message dicts."""- openai_messages = []-- for msg in messages:- # Handle both dict and Pydantic Message objects- if hasattr(msg, "role"):- role = msg.role- content = msg.content if hasattr(msg, "content") else ""- # Handle content that might be a list of content parts- if isinstance(content, list):- content = " ".join([c.text if hasattr(c, "text") else str(c) for c in content])- elif content is None:- content = ""- tool_calls = getattr(msg, "tool_calls", None)- tool_call_id = getattr(msg, "tool_call_id", None)- name = getattr(msg, "name", None)- else:- role = msg.get("role", "user")- content = msg.get("content", "")- tool_calls = msg.get("tool_calls", None)- tool_call_id = msg.get("tool_call_id", None)- name = msg.get("name", None)-- openai_msg = {"role": role, "content": content}-- if tool_calls:- # Convert tool calls to OpenAI format- openai_tool_calls = []- for tc in tool_calls:- if hasattr(tc, "function"):- tc_dict = {- "id": getattr(tc, "id", f"call_{uuid.uuid4().hex[:8]}"),- "type": "function",- "function": {- "name": tc.function.name,- "arguments": tc.function.arguments- if isinstance(tc.function.arguments, str)- else json.dumps(tc.function.arguments),- },- }- else:- tc_dict = {- "id": tc.get("id", f"call_{uuid.uuid4().hex[:8]}"),- "type": "function",- "function": tc.get("function", {}),- }- openai_tool_calls.append(tc_dict)- openai_msg["tool_calls"] = openai_tool_calls-- if tool_call_id:- openai_msg["tool_call_id"] = tool_call_id-- if name and role == "tool":- openai_msg["name"] = name-- openai_messages.append(openai_msg)-- return openai_messages-- def _convert_tools_to_openai_format(self, tools: list) -> list[dict]:- """Convert tools to OpenAI format for tokenizer."""- openai_tools = []- for tool in tools:- if isinstance(tool, dict):- # Already a dict, ensure it's in the right format- if "function" in tool:- openai_tools.append(tool)- else:- # Might be the function directly- openai_tools.append({"type": "function", "function": tool})- else:- # Convert object to dict- func = getattr(tool, "function", tool)- tool_dict = {+def _to_openai_tools(tools: list) -> list[dict]:+ """Convert tool objects to OpenAI-format dicts."""+ result = []+ for tool in tools:+ if isinstance(tool, dict):+ result.append(tool if "function" in tool else {"type": "function", "function": tool})+ else:+ func = getattr(tool, "function", tool)+ result.append(+ {"type": "function","function": {"name": getattr(func, "name", ""),@@ -221,138 +174,117 @@ class SGLangNativeAdapter(SimpleLLMRequestAdapter):"parameters": getattr(func, "parameters", {}),},}- openai_tools.append(tool_dict)- return openai_tools-- def _format_messages_to_text(self, messages: list, tools: list) -> str:- """- Format messages to text using tokenizer's apply_chat_template if available.+ )+ return result- Falls back to manual formatting if tokenizer is not available.- """- tokenizer = self._get_tokenizer()- if tokenizer is not None:- # Use tokenizer's apply_chat_template for proper formatting- openai_messages = self._convert_messages_to_openai_format(messages)- openai_tools = self._convert_tools_to_openai_format(tools) if tools else None+def _parse_glm47_tool_calls(text: str) -> list[ToolCall]:+ """Parse GLM-4.7 XML tool call format inline (no sglang dependency).+ Format: <tool_call>func_name<arg_key>k</arg_key><arg_value>v</arg_value>...</tool_call>+ """+ tool_calls = []+ for inner in re.findall(r"<tool_call>(.*?)</tool_call>", text, re.DOTALL):+ inner = inner.strip()+ # JSON format fallback+ if inner.startswith("{"):try:- formatted = tokenizer.apply_chat_template(- openai_messages,- tokenize=False,- add_generation_prompt=True,- tools=openai_tools,+ data = json.loads(inner)+ args = data.get("arguments", {})+ if isinstance(args, dict):+ args = json.dumps(args)+ tool_calls.append(+ ToolCall(+ id=f"call_{uuid.uuid4().hex[:8]}",+ type="function",+ function=FunctionCall(name=data.get("name", ""), arguments=args),+ ))- logger.debug(f"Formatted prompt using tokenizer ({len(formatted)} chars)")- return formatted- except Exception as e:- logger.warning(f"apply_chat_template failed: {e}, falling back to manual formatting")-- # Fallback to manual formatting- return self._format_messages_to_text_manual(messages, tools)-- def _format_messages_to_text_manual(self, messages: list, tools: list) -> str:- """Manual fallback formatting for when tokenizer is not available."""- formatted_parts = []- tools_section = self._format_tools_for_prompt(tools)-- for msg in messages:- # Handle both dict and Pydantic Message objects- if hasattr(msg, "role"):- role = msg.role- content = msg.content if hasattr(msg, "content") else ""- if isinstance(content, list):- content = " ".join([c.text if hasattr(c, "text") else str(c) for c in content])- elif content is None:- content = ""- tool_calls = getattr(msg, "tool_calls", None)- else:- role = msg.get("role", "user")- content = msg.get("content", "")- tool_calls = msg.get("tool_calls", None)-- if role == "system":- system_content = content + tools_section if tools_section else content- formatted_parts.append(f"<|im_start|>system\n{system_content}<|im_end|>")- tools_section = ""- elif role == "user":- formatted_parts.append(f"<|im_start|>user\n{content}<|im_end|>")- elif role == "assistant":- if tool_calls:- tc_parts = []- for tc in tool_calls:- if hasattr(tc, "function"):- tc_name = tc.function.name- tc_args = tc.function.arguments- else:- tc_name = tc.get("function", {}).get("name", "")- tc_args = tc.get("function", {}).get("arguments", "{}")-- if isinstance(tc_args, str):- try:- tc_args = json.loads(tc_args)- except Exception:- pass-- tc_parts.append(f'<tool_call>\n{{"name": "{tc_name}", "arguments": {json.dumps(tc_args)}}}\n</tool_call>')-- assistant_content = content + "\n" + "\n".join(tc_parts) if content else "\n".join(tc_parts)- formatted_parts.append(f"<|im_start|>assistant\n{assistant_content}<|im_end|>")- elif content:- formatted_parts.append(f"<|im_start|>assistant\n{content}<|im_end|>")- elif role == "tool":- formatted_parts.append(f"<|im_start|>user\n<tool_response>\n{content}\n</tool_response><|im_end|>")-- formatted_parts.append("<|im_start|>assistant\n")- return "\n".join(formatted_parts)-- def _parse_tool_calls(self, text: str) -> list[ToolCall]:- """- Parse tool calls from response text.-- Looks for patterns like:- <tool_call>- {"name": "tool_name", "arguments": {...}}- </tool_call>- """- tool_calls = []-- # Find all tool_call blocks- pattern = r"<tool_call>\s*(\{.*?\})\s*</tool_call>"- matches = re.findall(pattern, text, re.DOTALL)+ continue+ except json.JSONDecodeError:+ pass+ # GLM-4.7 XML format: first non-tag text is function name+ name_match = re.match(r"^([^<\n]+)", inner)+ if not name_match:+ continue+ func_name = name_match.group(1).strip()+ keys = re.findall(r"<arg_key>(.*?)</arg_key>", inner, re.DOTALL)+ vals = re.findall(r"<arg_value>(.*?)</arg_value>", inner, re.DOTALL)+ tool_calls.append(+ ToolCall(+ id=f"call_{uuid.uuid4().hex[:8]}",+ type="function",+ function=FunctionCall(name=func_name, arguments=json.dumps(dict(zip(keys, vals)))),+ )+ )+ return tool_calls- for match in matches:- try:- tc_data = json.loads(match)- name = tc_data.get("name", "")- arguments = tc_data.get("arguments", {})- if isinstance(arguments, dict):- arguments = json.dumps(arguments)+def _parse_tool_calls(text: str, tools: list | None = None, tool_call_parser: str = "glm47") -> list[ToolCall]:+ """Parse tool calls from response text.- tool_call = ToolCall(+ Tries SGLang's FunctionCallParser first; falls back to inline GLM-4.7 parser.+ """+ try:+ from sglang.srt.function_call.core_types import Function, Tool+ from sglang.srt.function_call.function_call_parser import FunctionCallParser++ sglang_tools: list[Tool] = []+ for t in tools or []:+ func = t.get("function", t) if isinstance(t, dict) else getattr(t, "function", t)+ name = func.get("name", "") if isinstance(func, dict) else getattr(func, "name", "")+ desc = func.get("description", "") if isinstance(func, dict) else getattr(func, "description", "")+ params = func.get("parameters", {}) if isinstance(func, dict) else getattr(func, "parameters", {})+ sglang_tools.append(Tool(type="function", function=Function(name=name, description=desc, parameters=params)))++ parser = FunctionCallParser(tools=sglang_tools, tool_call_parser=tool_call_parser)+ result = parser.detector.detect_and_parse(text, sglang_tools)+ tool_calls = []+ for call in result.calls:+ arguments = call.parameters if isinstance(call.parameters, str) else json.dumps(call.parameters)+ tool_calls.append(+ ToolCall(id=f"call_{uuid.uuid4().hex[:8]}",type="function",- function=FunctionCall(- name=name,- arguments=arguments,- ),+ function=FunctionCall(name=call.name, arguments=arguments),)- tool_calls.append(tool_call)- except json.JSONDecodeError as e:- logger.warning(f"Failed to parse tool call JSON: {e}")- continue-+ )return tool_calls+ except ImportError:+ pass+ except Exception as e:+ logger.warning(f"SGLang tool call parser ({tool_call_parser}) failed: {e}")++ # Inline GLM-4.7 fallback (no sglang required)+ return _parse_glm47_tool_calls(text)+++def _strip_tool_calls(text: str) -> str:+ return re.sub(r"<tool_call>.*?</tool_call>", "", text, flags=re.DOTALL).strip()+++class SGLangNativeAdapter(SimpleLLMRequestAdapter):+ """+ Adapter using SGLang's native /generate endpoint for multi-turn RL training.- def _extract_content_without_tool_calls(self, text: str) -> str:- """Extract content from response, removing tool_call blocks."""- # Remove tool_call blocks- cleaned = re.sub(r"<tool_call>.*?</tool_call>", "", text, flags=re.DOTALL)- # Clean up whitespace- cleaned = cleaned.strip()- return cleaned+ Flow:+ 1. Apply model's chat template via HF tokenizer → input_ids (single tokenization step)+ 2. POST input_ids to SGLang /generate → output_ids + output_token_logprobs+ 3. Parse tool calls from response text+ """++ def __init__(self, *args, model_settings: ModelSettingsUnion | None = None, **kwargs):+ self.model_settings = model_settings+ super().__init__(*args, **kwargs)+ self._sglang_client: Optional[SGLangNativeClient] = None++ def _get_sglang_client(self) -> SGLangNativeClient:+ if self._sglang_client is None:+ base_url = (self.llm_config.model_endpoint or "").rstrip("/")+ if base_url.endswith("/v1"):+ base_url = base_url[:-3]+ self._sglang_client = SGLangNativeClient(base_url=base_url, api_key=None)+ return self._sglang_clientasync def invoke_llm(self,@@ -364,98 +296,73 @@ class SGLangNativeAdapter(SimpleLLMRequestAdapter):step_id: str | None = None,actor: str | None = None,) -> AsyncGenerator[LettaMessage | None, None]:- """- Execute LLM request using SGLang native endpoint.-- This method:- 1. Formats messages and tools to text using chat template- 2. Calls SGLang native /generate endpoint- 3. Extracts output_ids and output_token_logprobs- 4. Parses tool calls from response- 5. Converts response to standard format- """self.request_data = request_data- # Get sampling params from request_datasampling_params = {"temperature": request_data.get("temperature", 0.7),"max_new_tokens": request_data.get("max_tokens", 4096),"top_p": request_data.get("top_p", 0.9),}- # Format messages to text (includes tools in prompt)- text_input = self._format_messages_to_text(messages, tools)+ # Tokenize via HF apply_chat_template — correct template, no double tokenization+ openai_msgs = _to_openai_messages(messages)+ logger.info(f"SGLang native input: {len(openai_msgs)} messages, roles={[m['role'] for m in openai_msgs]}")+ if openai_msgs:+ first_content = openai_msgs[0].get("content", "")+ logger.info(f" first msg content[:200]: {str(first_content)[:200]}")+ input_ids = _messages_to_input_ids(self.llm_config.model, messages, tools)- # Call SGLang native endpointclient = self._get_sglang_client()-- try:- response = await client.generate(- text=text_input,- sampling_params=sampling_params,- return_logprob=True,- )- except Exception as e:- logger.error(f"SGLang native endpoint error: {e}")- raise+ response = await client.generate(+ input_ids=input_ids,+ sampling_params=sampling_params,+ return_logprob=True,+ )self.llm_request_finish_timestamp_ns = get_utc_timestamp_ns()-- # Store native response dataself.response_data = response- # Extract SGLang native dataself.output_ids = response.get("output_ids")- # output_token_logprobs is inside meta_infometa_info = response.get("meta_info", {})self.output_token_logprobs = meta_info.get("output_token_logprobs")- # Extract text responsetext_response = response.get("text", "")+ finish_reason_raw = response.get("meta_info", {}).get("finish_reason", {})+ output_ids_raw = response.get("output_ids", [])+ logger.info(f"SGLang raw response[:300]: {repr(text_response[:300])}")+ logger.info(f"SGLang finish_reason: {finish_reason_raw}")+ logger.info(f"SGLang output_ids[:15]: {output_ids_raw[:15]}")+ # Decode output_ids directly to verify what tokens were generated+ try:+ tok = _get_tokenizer(_resolve_tokenizer_path(self.llm_config.model))+ logger.info(f"SGLang output decoded[:200]: {repr(tok.decode(output_ids_raw[:50]))}")+ except Exception as _e:+ pass- # Remove trailing end token if present- if text_response.endswith("<|im_end|>"):- text_response = text_response[:-10]-- # Parse tool calls from response- parsed_tool_calls = self._parse_tool_calls(text_response)-- # Extract content (text without tool_call blocks)- content_text = self._extract_content_without_tool_calls(text_response)+ tool_call_parser = "qwen25"+ if self.model_settings is not None and getattr(self.model_settings, "provider_type", None) == ProviderType.sglang:+ tool_call_parser = getattr(self.model_settings, "tool_call_parser", None) or tool_call_parser+ parsed_tool_calls = _parse_tool_calls(text_response, tools=tools, tool_call_parser=tool_call_parser)+ content_text = _strip_tool_calls(text_response)- # Determine finish reason- meta_info = response.get("meta_info", {})finish_reason_info = meta_info.get("finish_reason", {})- if isinstance(finish_reason_info, dict):- finish_reason = finish_reason_info.get("type", "stop")- else:- finish_reason = "stop"-- # If we have tool calls, set finish_reason to tool_calls+ finish_reason = finish_reason_info.get("type", "stop") if isinstance(finish_reason_info, dict) else "stop"if parsed_tool_calls:finish_reason = "tool_calls"- # Convert to standard ChatCompletionResponse format for compatibility- # Build logprobs in OpenAI format from SGLang formatlogprobs_content = Noneif self.output_token_logprobs:- logprobs_content = []- for i, lp_data in enumerate(self.output_token_logprobs):- # SGLang format: [logprob, token_id, top_logprob]- logprob = lp_data[0] if len(lp_data) > 0 else 0.0- token_id = lp_data[1] if len(lp_data) > 1 else 0- logprobs_content.append(- ChatCompletionTokenLogprob(- token=str(token_id),- logprob=logprob,- bytes=None,- top_logprobs=[],- )+ logprobs_content = [+ ChatCompletionTokenLogprob(+ token=str(lp[1]) if len(lp) > 1 else "0",+ logprob=lp[0] if len(lp) > 0 else 0.0,+ bytes=None,+ top_logprobs=[],)+ for lp in self.output_token_logprobs+ ]choice_logprobs = ChoiceLogprobs(content=logprobs_content) if logprobs_content else None-- # Build chat completion responseprompt_tokens = meta_info.get("prompt_tokens", 0)completion_tokens = len(self.output_ids) if self.output_ids else 0@@ -468,8 +375,8 @@ class SGLangNativeAdapter(SimpleLLMRequestAdapter):index=0,message=ChoiceMessage(role="assistant",- content=content_text if content_text else None,- tool_calls=parsed_tool_calls if parsed_tool_calls else None,+ content=content_text or None,+ tool_calls=parsed_tool_calls or None,),logprobs=choice_logprobs,)@@ -481,23 +388,12 @@ class SGLangNativeAdapter(SimpleLLMRequestAdapter):),)- # Extract content- if content_text:- self.content = [TextContent(text=content_text)]- else:- self.content = None-- # No reasoning content from native endpoint+ self.content = [TextContent(text=content_text)] if content_text else Noneself.reasoning_content = None-- # Set tool callsself.tool_calls = parsed_tool_callsself.tool_call = parsed_tool_calls[0] if parsed_tool_calls else None-- # Set logprobsself.logprobs = choice_logprobs- # Extract usage statisticsself.usage.step_count = 1self.usage.completion_tokens = completion_tokensself.usage.prompt_tokens = prompt_tokens@@ -506,7 +402,7 @@ class SGLangNativeAdapter(SimpleLLMRequestAdapter):self.log_provider_trace(step_id=step_id, actor=actor)logger.info(- f"SGLang native response: {len(self.output_ids or [])} tokens, "+ f"SGLang native: {len(self.output_ids or [])} output tokens, "f"{len(self.output_token_logprobs or [])} logprobs, "f"{len(parsed_tool_calls)} tool calls")diff --git a/letta/agents/letta_agent_v3.py b/letta/agents/letta_agent_v3.pyindex cbc3b4459..3addbcb6b 100644--- a/letta/agents/letta_agent_v3.py+++ b/letta/agents/letta_agent_v3.py@@ -282,11 +282,13 @@ class LettaAgentV3(LettaAgentV2):self.in_context_messages = curr_in_context_messages- # Check if we should use SGLang native adapter for multi-turn RL training+ # Check if we should use SGLang native adapter for multi-turn RL training.+ # Matches handles starting with "sglang/" OR providers named like "*sglang*"+ # (e.g. "slime-sglang" used in training).+ _handle = self.agent_state.llm_config.handle or ""+ _provider = (self.agent_state.llm_config.provider_name or "").lower()use_sglang_native = (- self.agent_state.llm_config.return_token_ids- and self.agent_state.llm_config.handle- and self.agent_state.llm_config.handle.startswith("sglang/")+ self.agent_state.llm_config.return_token_ids and _handle and (_handle.startswith("sglang/") or "sglang" in _provider))self.return_token_ids = use_sglang_native@@ -295,6 +297,7 @@ class LettaAgentV3(LettaAgentV2):llm_adapter = SGLangNativeAdapter(llm_client=self.llm_client,llm_config=self.agent_state.llm_config,+ model_settings=self.agent_state.model_settings,call_type=LLMCallType.agent_step,agent_id=self.agent_state.id,agent_tags=self.agent_state.tags,@@ -515,6 +518,7 @@ class LettaAgentV3(LettaAgentV2):llm_adapter = SGLangNativeAdapter(llm_client=self.llm_client,llm_config=self.agent_state.llm_config,+ model_settings=self.agent_state.model_settings,call_type=LLMCallType.agent_step,agent_id=self.agent_state.id,agent_tags=self.agent_state.tags,diff --git a/letta/llm_api/sglang_native_client.py b/letta/llm_api/sglang_native_client.pyindex ade9adc35..3f2d9af63 100644--- a/letta/llm_api/sglang_native_client.py+++ b/letta/llm_api/sglang_native_client.py@@ -1,15 +1,11 @@"""SGLang Native Client for Letta.-This client uses SGLang's native /generate endpoint instead of the OpenAI-compatible-/v1/chat/completions endpoint. The native endpoint returns token IDs and per-token-logprobs, which are essential for multi-turn RL training.--The OpenAI-compatible endpoint only returns token strings, not IDs, making it-impossible to accurately reconstruct the token sequence for training.+Uses SGLang's native /generate endpoint with input_ids for exact token-level control.+Returns token IDs and per-token logprobs essential for multi-turn RL training."""-from typing import Any, Dict, Optional+from typing import Any, Dict, List, Optionalimport httpx@@ -21,70 +17,50 @@ logger = get_logger(__name__)class SGLangNativeClient:"""Client for SGLang's native /generate endpoint.- Unlike the OpenAI-compatible endpoint, this returns:+ Uses input_ids instead of text to avoid double tokenization and ensure+ the correct chat template is applied (via HF tokenizer on the caller side).++ Returns:- output_ids: List of token IDs- output_token_logprobs: List of [logprob, token_id, top_logprob] tuples-- This is essential for RL training where we need exact token IDs, not re-tokenized text."""def __init__(self, base_url: str, api_key: Optional[str] = None):- """- Initialize the SGLang native client.-- Args:- base_url: Base URL for SGLang server (e.g., http://localhost:30000)- api_key: Optional API key for authentication- """- # Remove /v1 suffix if present - native endpoint is at rootself.base_url = base_url.rstrip("/")if self.base_url.endswith("/v1"):self.base_url = self.base_url[:-3]self.api_key = api_key+ @property+ def _headers(self) -> dict:+ headers = {"Content-Type": "application/json"}+ if self.api_key:+ headers["Authorization"] = f"Bearer {self.api_key}"+ return headers+async def generate(self,- text: str,+ input_ids: List[int],sampling_params: Optional[Dict[str, Any]] = None,return_logprob: bool = True,) -> Dict[str, Any]:- """- Call SGLang's native /generate endpoint.+ """Call SGLang's native /generate endpoint with pre-tokenized input.Args:- text: The formatted prompt text (with chat template applied)- sampling_params: Sampling parameters (temperature, max_new_tokens, etc.)- return_logprob: Whether to return logprobs (default True for RL training)+ input_ids: Token IDs from apply_chat_template(tokenize=True)+ sampling_params: temperature, max_new_tokens, top_p, etc.+ return_logprob: Whether to return per-token logprobs (default True for RL)Returns:- Response dict with:- - text: Generated text- - output_ids: List of token IDs- - output_token_logprobs: List of [logprob, token_id, top_logprob] tuples- - meta_info: Metadata including finish_reason, prompt_tokens, etc.-- Example response:{- "text": "Hello! How can I help?",- "output_ids": [9707, 0, 2585, 646, 358, 1492, 30],- "output_token_logprobs": [- [-0.005, 9707, null],- [0.0, 0, null],- ...- ],- "meta_info": {- "finish_reason": {"type": "stop", "matched": 151645},- "prompt_tokens": 42,- ...- }+ "text": "...",+ "output_ids": [token_id, ...],+ "output_token_logprobs": [[logprob, token_id, top_logprob], ...],+ "meta_info": {"finish_reason": ..., "prompt_tokens": ..., ...}}"""- headers = {"Content-Type": "application/json"}- if self.api_key:- headers["Authorization"] = f"Bearer {self.api_key}"-payload = {- "text": text,+ "input_ids": input_ids,"sampling_params": sampling_params or {},"return_logprob": return_logprob,}@@ -93,7 +69,7 @@ class SGLangNativeClient:response = await client.post(f"{self.base_url}/generate",json=payload,- headers=headers,+ headers=self._headers,)response.raise_for_status()return response.json()diff --git a/letta/schemas/llm_config.py b/letta/schemas/llm_config.pyindex 7106ed319..3e74b7b71 100644--- a/letta/schemas/llm_config.py+++ b/letta/schemas/llm_config.py@@ -129,6 +129,11 @@ class LLMConfig(BaseModel):description="Whether to return token IDs for all LLM generations via SGLang native endpoint. ""Required for multi-turn RL training with loss masking. Only works with SGLang provider.",)+ tool_call_parser: Optional[str] = Field(+ None,+ description="SGLang tool call parser name (e.g. 'glm47', 'qwen25', 'hermes'). "+ "Used by the SGLang native adapter to parse tool calls from raw model output.",+ )@model_validator(mode="before")@classmethod@@ -367,12 +372,23 @@ class LLMConfig(BaseModel):OpenAIModelSettings,OpenAIReasoning,OpenRouterModelSettings,+ SGLangModelSettings,TogetherModelSettings,XAIModelSettings,ZAIModelSettings,)if self.model_endpoint_type == "openai":+ handle = self.handle or ""+ provider_name = (self.provider_name or "").lower()+ if handle.startswith("sglang/") or "sglang" in provider_name:+ return SGLangModelSettings(+ max_output_tokens=self.max_tokens or 4096,+ temperature=self.temperature,+ reasoning=OpenAIReasoning(reasoning_effort=self.reasoning_effort or "minimal"),+ strict=self.strict,+ tool_call_parser=self.tool_call_parser,+ )return OpenAIModelSettings(max_output_tokens=self.max_tokens or 4096,temperature=self.temperature,diff --git a/letta/schemas/model.py b/letta/schemas/model.pyindex 621bd4f09..d143b9d3c 100644--- a/letta/schemas/model.py+++ b/letta/schemas/model.py@@ -130,6 +130,7 @@ class Model(LLMConfig, ModelBase):"""Returns the JSON schema for the ModelSettings class corresponding to this model's provider."""PROVIDER_SETTINGS_MAP = {ProviderType.openai: OpenAIModelSettings,+ ProviderType.sglang: SGLangModelSettings,ProviderType.anthropic: AnthropicModelSettings,ProviderType.google_ai: GoogleAIModelSettings,ProviderType.google_vertex: GoogleVertexModelSettings,@@ -257,6 +258,21 @@ class OpenAIModelSettings(ModelSettings):}+class SGLangModelSettings(OpenAIModelSettings):+ """SGLang model configuration (OpenAI-compatible runtime with SGLang-specific parsing)."""++ provider_type: Literal[ProviderType.sglang] = Field(ProviderType.sglang, description="The type of the provider.")+ tool_call_parser: Optional[str] = Field(+ None,+ description="SGLang tool call parser name (for example 'glm47', 'qwen25', or 'hermes').",+ )++ def _to_legacy_config_params(self) -> dict:+ params = super()._to_legacy_config_params()+ params["tool_call_parser"] = self.tool_call_parser+ return params++# "thinking": {# "type": "enabled",# "budget_tokens": 10000@@ -516,6 +532,7 @@ class ChatGPTOAuthModelSettings(ModelSettings):ModelSettingsUnion = Annotated[Union[OpenAIModelSettings,+ SGLangModelSettings,AnthropicModelSettings,GoogleAIModelSettings,GoogleVertexModelSettings,diff --git a/letta/schemas/providers/sglang.py b/letta/schemas/providers/sglang.pyindex 48d5d0130..31f33a8f5 100644--- a/letta/schemas/providers/sglang.py+++ b/letta/schemas/providers/sglang.py@@ -44,7 +44,7 @@ class SGLangProvider(Provider):configs.append(LLMConfig(model=model_name,- model_endpoint_type="openai", # SGLang is OpenAI-compatible+ model_endpoint_type="openai", # SGLang exposes OpenAI-compatible APImodel_endpoint=base_url,model_wrapper=self.default_prompt_formatter,context_window=model.get("max_model_len", 32768),@@ -52,6 +52,7 @@ class SGLangProvider(Provider):max_tokens=self.get_default_max_output_tokens(model_name),provider_name=self.name,provider_category=self.provider_category,+))diff --git a/letta/services/agent_manager.py b/letta/services/agent_manager.pyindex 364f306da..73af54829 100644--- a/letta/services/agent_manager.py+++ b/letta/services/agent_manager.py@@ -515,7 +515,6 @@ class AgentManager:# Will set mode-specific default prompt if no prompt is providedeffective_compaction_settings = effective_compaction_settings.set_mode_specific_prompt()-new_agent = AgentModel(name=agent_create.name,system=derive_system_message(diff --git a/letta/services/provider_manager.py b/letta/services/provider_manager.pyindex 011577b48..b59f3846e 100644--- a/letta/services/provider_manager.py+++ b/letta/services/provider_manager.py@@ -1062,7 +1062,10 @@ class ProviderManager:else:model_endpoint = f"https://api.{provider.provider_type.value}.com/v1"- # Construct the LLMConfig from the model and provider data+ # Construct the LLMConfig from the model and provider data.+ # SGLang providers get return_token_ids/return_logprobs=True so the native+ # adapter is used and token IDs are returned for RL training.+ is_sglang = provider.provider_type == ProviderType.sglangllm_config = LLMConfig(model=model.name,model_endpoint_type=model.model_endpoint_type,@@ -1072,6 +1075,8 @@ class ProviderManager:provider_name=provider.name,provider_category=provider.provider_category,max_tokens=max_tokens,+ return_token_ids=is_sglang,+ return_logprobs=is_sglang,)return llm_configdiff --git a/tests/managers/test_agent_manager.py b/tests/managers/test_agent_manager.pyindex 4e60875e5..4456f59e3 100644--- a/tests/managers/test_agent_manager.py+++ b/tests/managers/test_agent_manager.py@@ -1954,6 +1954,7 @@ async def test_agent_state_schema_unchanged(server: SyncServer):"return_logprobs","top_logprobs","return_token_ids",+ "tool_call_parser",}actual_llm_config_fields = set(llm_config_fields.keys())if actual_llm_config_fields != expected_llm_config_fields: