Platform for stateful agents: AI with advanced memory that can learn and self-improve over time.
fix: manually count tokens for streaming lmstudio models
jnjpng6b082f044728
diff --git a/letta/agents/letta_agent.py b/letta/agents/letta_agent.pyindex 7d2f5b86f..b5a4eac82 100644--- a/letta/agents/letta_agent.py+++ b/letta/agents/letta_agent.py@@ -757,6 +757,9 @@ class LettaAgent(BaseAgent):interface = OpenAIStreamingInterface(use_assistant_message=use_assistant_message,put_inner_thoughts_in_kwarg=agent_state.llm_config.put_inner_thoughts_in_kwargs,+ is_openai_proxy=agent_state.llm_config.provider_name == "lmstudio_openai",+ messages=current_in_context_messages + new_in_context_messages,+ tools=request_data.get("tools", []),)else:raise ValueError(f"Streaming not supported for {agent_state.llm_config}")@@ -782,13 +785,20 @@ class LettaAgent(BaseAgent):stream_end_time_ns = get_utc_timestamp_ns()- # update usage+ # Some providers that rely on the OpenAI client currently e.g. LMStudio don't get usage metrics back on the last streaming chunk, fall back to manual values+ if isinstance(interface, OpenAIStreamingInterface) and not interface.input_tokens and not interface.output_tokens:+ logger.warning(+ f"No token usage metrics received from OpenAI streaming interface for {agent_state.llm_config.model}, falling back to estimated values. Input tokens: {interface.fallback_input_tokens}, Output tokens: {interface.fallback_output_tokens}"+ )+ interface.input_tokens = interface.fallback_input_tokens+ interface.output_tokens = interface.fallback_output_tokens+usage.step_count += 1usage.completion_tokens += interface.output_tokensusage.prompt_tokens += interface.input_tokensusage.total_tokens += interface.input_tokens + interface.output_tokensMetricRegistry().message_output_tokens.record(- interface.output_tokens, dict(get_ctx_attributes(), **{"model.name": agent_state.llm_config.model})+ usage.completion_tokens, dict(get_ctx_attributes(), **{"model.name": agent_state.llm_config.model}))# log LLM request time@@ -815,9 +825,9 @@ class LettaAgent(BaseAgent):agent_state,tool_rules_solver,UsageStatistics(- completion_tokens=interface.output_tokens,- prompt_tokens=interface.input_tokens,- total_tokens=interface.input_tokens + interface.output_tokens,+ completion_tokens=usage.completion_tokens,+ prompt_tokens=usage.prompt_tokens,+ total_tokens=usage.total_tokens,),reasoning_content=reasoning_content,pre_computed_assistant_message_id=interface.letta_message_id,@@ -862,8 +872,8 @@ class LettaAgent(BaseAgent):# "stop_sequence": None,"type": "message","usage": {- "input_tokens": interface.input_tokens,- "output_tokens": interface.output_tokens,+ "input_tokens": usage.prompt_tokens,+ "output_tokens": usage.completion_tokens,},},step_id=step_id,diff --git a/letta/interfaces/openai_streaming_interface.py b/letta/interfaces/openai_streaming_interface.pyindex f03e81f53..aee3584d3 100644--- a/letta/interfaces/openai_streaming_interface.py+++ b/letta/interfaces/openai_streaming_interface.py@@ -9,6 +9,7 @@ from openai.types.chat.chat_completion_chunk import ChatCompletionChunkfrom letta.constants import DEFAULT_MESSAGE_TOOL, DEFAULT_MESSAGE_TOOL_KWARGfrom letta.helpers.datetime_helpers import get_utc_timestamp_ns, ns_to_msfrom letta.llm_api.openai_client import is_openai_reasoning_model+from letta.local_llm.utils import num_tokens_from_functions, num_tokens_from_messagesfrom letta.log import get_loggerfrom letta.otel.context import get_ctx_attributesfrom letta.otel.metric_registry import MetricRegistry@@ -19,6 +20,7 @@ from letta.schemas.message import Messagefrom letta.schemas.openai.chat_completion_response import FunctionCall, ToolCallfrom letta.server.rest_api.json_parser import OptimisticJSONParserfrom letta.streaming_utils import JSONInnerThoughtsExtractor+from letta.utils import count_tokenslogger = get_logger(__name__)@@ -30,7 +32,14 @@ class OpenAIStreamingInterface:and detection of tool call events."""- def __init__(self, use_assistant_message: bool = False, put_inner_thoughts_in_kwarg: bool = False):+ def __init__(+ self,+ use_assistant_message: bool = False,+ put_inner_thoughts_in_kwarg: bool = False,+ is_openai_proxy: bool = False,+ messages: Optional[list] = None,+ tools: Optional[list] = None,+ ):self.use_assistant_message = use_assistant_messageself.assistant_message_tool_name = DEFAULT_MESSAGE_TOOLself.assistant_message_tool_kwarg = DEFAULT_MESSAGE_TOOL_KWARG@@ -53,10 +62,19 @@ class OpenAIStreamingInterface:self.message_id = Noneself.model = None- # token counters+ # Token counters (from OpenAI usage)self.input_tokens = 0self.output_tokens = 0+ # Fallback token counters (using tiktoken cl200k-base)+ self.fallback_input_tokens = 0+ self.fallback_output_tokens = 0++ # Store messages and tools for fallback counting+ self.is_openai_proxy = is_openai_proxy+ self.messages = messages or []+ self.tools = tools or []+self.content_buffer: list[str] = []self.tool_call_name: str | None = Noneself.tool_call_id: str | None = None@@ -95,6 +113,18 @@ class OpenAIStreamingInterface:Iterates over the OpenAI stream, yielding SSE events.It also collects tokens and detects if a tool call is triggered."""+ # Fallback input token counting - this should only be required for non-OpenAI providers using the OpenAI client (e.g. LMStudio)+ if self.is_openai_proxy:+ if self.messages:+ # Convert messages to dict format for token counting+ message_dicts = [msg.to_openai_dict() if hasattr(msg, "to_openai_dict") else msg for msg in self.messages]+ self.fallback_input_tokens = num_tokens_from_messages(message_dicts) # fallback to gpt-4 cl100k-base++ if self.tools:+ # Convert tools to dict format for token counting+ tool_dicts = [tool["function"] if isinstance(tool, dict) and "function" in tool else tool for tool in self.tools]+ self.fallback_input_tokens += num_tokens_from_functions(tool_dicts)+first_chunk = Truetry:async with stream:@@ -113,6 +143,9 @@ class OpenAIStreamingInterface:metric_attributes["model.name"] = chunk.modelMetricRegistry().ttft_ms_histogram.record(ns_to_ms(ttft_ns), metric_attributes)+ if self.is_openai_proxy:+ self.fallback_output_tokens += count_tokens(chunk.model_dump_json())+first_chunk = Falseif not self.model or not self.message_id:@@ -153,6 +186,9 @@ class OpenAIStreamingInterface:tool_call.function.arguments)+ if self.is_openai_proxy:+ self.fallback_output_tokens += count_tokens(tool_call.function.arguments)+# If we have inner thoughts, we should output them as a chunkif updates_inner_thoughts:if prev_message_type and prev_message_type != "reasoning_message":diff --git a/tests/configs/llm_model_configs/bartowski-ministral-8b-instruct-2410.json b/tests/configs/llm_model_configs/bartowski-ministral-8b-instruct-2410.jsonnew file mode 100644index 000000000..c2baa66ac--- /dev/null+++ b/tests/configs/llm_model_configs/bartowski-ministral-8b-instruct-2410.json@@ -0,0 +1,8 @@+{+ "context_window": 8192,+ "model": "ministral-8b-instruct-2410",+ "model_endpoint_type": "openai",+ "model_endpoint": "http://localhost:1234/v1",+ "model_wrapper": null,+ "provider_name": "lmstudio_openai"+}diff --git a/tests/configs/llm_model_configs/lmstudio-meta-llama-3.1-8b-instruct.json b/tests/configs/llm_model_configs/lmstudio-meta-llama-3.1-8b-instruct.jsonnew file mode 100644index 000000000..fdf80602e--- /dev/null+++ b/tests/configs/llm_model_configs/lmstudio-meta-llama-3.1-8b-instruct.json@@ -0,0 +1,8 @@+{+ "context_window": 8192,+ "model": "meta-llama-3.1-8b-instruct",+ "model_endpoint_type": "openai",+ "model_endpoint": "http://localhost:1234/v1",+ "model_wrapper": null,+ "provider_name": "lmstudio_openai"+}diff --git a/tests/configs/llm_model_configs/lmstudio-qwen-2.5-7b-instruct.json b/tests/configs/llm_model_configs/lmstudio-qwen-2.5-7b-instruct.jsonnew file mode 100644index 000000000..a737f1a55--- /dev/null+++ b/tests/configs/llm_model_configs/lmstudio-qwen-2.5-7b-instruct.json@@ -0,0 +1,8 @@+{+ "context_window": 8192,+ "model": "qwen2.5-7b-instruct",+ "model_endpoint_type": "openai",+ "model_endpoint": "http://localhost:1234/v1",+ "model_wrapper": null,+ "provider_name": "lmstudio_openai"+}diff --git a/tests/configs/llm_model_configs/mlx-meta-llama-3.1-8b-instruct-8bit.json b/tests/configs/llm_model_configs/mlx-meta-llama-3.1-8b-instruct-8bit.jsonnew file mode 100644index 000000000..f6be77495--- /dev/null+++ b/tests/configs/llm_model_configs/mlx-meta-llama-3.1-8b-instruct-8bit.json@@ -0,0 +1,8 @@+{+ "context_window": 8192,+ "model": "meta-llama-3.1-8b-instruct-8bit",+ "model_endpoint_type": "openai",+ "model_endpoint": "http://localhost:1234/v1",+ "model_wrapper": null,+ "provider_name": "lmstudio_openai"+}diff --git a/tests/configs/llm_model_configs/mlx-ministral-8b-instruct-2410.json b/tests/configs/llm_model_configs/mlx-ministral-8b-instruct-2410.jsonnew file mode 100644index 000000000..c2baa66ac--- /dev/null+++ b/tests/configs/llm_model_configs/mlx-ministral-8b-instruct-2410.json@@ -0,0 +1,8 @@+{+ "context_window": 8192,+ "model": "ministral-8b-instruct-2410",+ "model_endpoint_type": "openai",+ "model_endpoint": "http://localhost:1234/v1",+ "model_wrapper": null,+ "provider_name": "lmstudio_openai"+}diff --git a/tests/configs/llm_model_configs/mlx-qwen-2.5-7b-instruct.json b/tests/configs/llm_model_configs/mlx-qwen-2.5-7b-instruct.jsonnew file mode 100644index 000000000..50b15d56c--- /dev/null+++ b/tests/configs/llm_model_configs/mlx-qwen-2.5-7b-instruct.json@@ -0,0 +1,8 @@+{+ "context_window": 8192,+ "model": "qwen-2.5-7b-instruct",+ "model_endpoint_type": "openai",+ "model_endpoint": "http://localhost:1234/v1",+ "model_wrapper": null,+ "provider_name": "lmstudio_openai"+}diff --git a/tests/integration_test_send_message.py b/tests/integration_test_send_message.pyindex 314cf702f..2952e2fda 100644--- a/tests/integration_test_send_message.py+++ b/tests/integration_test_send_message.py@@ -131,6 +131,24 @@ reasoning_configs = ["openai-o1.json","openai-o3.json","openai-o4-mini.json",+ # "azure-gpt-4o-mini.json",+ # "claude-4-sonnet.json",+ # "claude-3-5-sonnet.json",+ # "claude-3-7-sonnet.json",+ # "claude-3-7-sonnet-extended.json",+ # "bedrock-claude-4-sonnet.json",+ # "gemini-1.5-pro.json",+ # "gemini-2.5-flash-vertex.json",+ # "gemini-2.5-pro-vertex.json",+ # "together-qwen-2.5-72b-instruct.json",+ # "ollama.json", # TODO (cliandy): enable this in ollama testing+ # TODO @jnjpng: not supported in CI yet, uncomment to test locally (requires lmstudio running locally with respective models loaded)+ # "lmstudio-meta-llama-3.1-8b-instruct.json",+ # "lmstudio-qwen-2.5-7b-instruct.json",+ # "mlx-qwen-2.5-7b-instruct.json",+ # "mlx-meta-llama-3.1-8b-instruct-8bit.json",+ # "mlx-ministral-8b-instruct-2410.json",+ # "bartowski-ministral-8b-instruct-2410.json"]