Skip to content

armgit / letta

Public Imported from GitHub
0

Platform for stateful agents: AI with advanced memory that can learn and self-improve over time.

fix: manually count tokens for streaming lmstudio models

jnjpng6b082f044728
diff --git a/letta/agents/letta_agent.py b/letta/agents/letta_agent.py
index 7d2f5b86f..b5a4eac82 100644
--- a/letta/agents/letta_agent.py
+++ b/letta/agents/letta_agent.py
@@ -757,6 +757,9 @@ class LettaAgent(BaseAgent):
interface = OpenAIStreamingInterface(
use_assistant_message=use_assistant_message,
put_inner_thoughts_in_kwarg=agent_state.llm_config.put_inner_thoughts_in_kwargs,
+ is_openai_proxy=agent_state.llm_config.provider_name == "lmstudio_openai",
+ messages=current_in_context_messages + new_in_context_messages,
+ tools=request_data.get("tools", []),
)
else:
raise ValueError(f"Streaming not supported for {agent_state.llm_config}")
@@ -782,13 +785,20 @@ class LettaAgent(BaseAgent):
stream_end_time_ns = get_utc_timestamp_ns()
- # update usage
+ # Some providers that rely on the OpenAI client currently e.g. LMStudio don't get usage metrics back on the last streaming chunk, fall back to manual values
+ if isinstance(interface, OpenAIStreamingInterface) and not interface.input_tokens and not interface.output_tokens:
+ logger.warning(
+ f"No token usage metrics received from OpenAI streaming interface for {agent_state.llm_config.model}, falling back to estimated values. Input tokens: {interface.fallback_input_tokens}, Output tokens: {interface.fallback_output_tokens}"
+ )
+ interface.input_tokens = interface.fallback_input_tokens
+ interface.output_tokens = interface.fallback_output_tokens
+
usage.step_count += 1
usage.completion_tokens += interface.output_tokens
usage.prompt_tokens += interface.input_tokens
usage.total_tokens += interface.input_tokens + interface.output_tokens
MetricRegistry().message_output_tokens.record(
- interface.output_tokens, dict(get_ctx_attributes(), **{"model.name": agent_state.llm_config.model})
+ usage.completion_tokens, dict(get_ctx_attributes(), **{"model.name": agent_state.llm_config.model})
)
# log LLM request time
@@ -815,9 +825,9 @@ class LettaAgent(BaseAgent):
agent_state,
tool_rules_solver,
UsageStatistics(
- completion_tokens=interface.output_tokens,
- prompt_tokens=interface.input_tokens,
- total_tokens=interface.input_tokens + interface.output_tokens,
+ completion_tokens=usage.completion_tokens,
+ prompt_tokens=usage.prompt_tokens,
+ total_tokens=usage.total_tokens,
),
reasoning_content=reasoning_content,
pre_computed_assistant_message_id=interface.letta_message_id,
@@ -862,8 +872,8 @@ class LettaAgent(BaseAgent):
# "stop_sequence": None,
"type": "message",
"usage": {
- "input_tokens": interface.input_tokens,
- "output_tokens": interface.output_tokens,
+ "input_tokens": usage.prompt_tokens,
+ "output_tokens": usage.completion_tokens,
},
},
step_id=step_id,
diff --git a/letta/interfaces/openai_streaming_interface.py b/letta/interfaces/openai_streaming_interface.py
index f03e81f53..aee3584d3 100644
--- a/letta/interfaces/openai_streaming_interface.py
+++ b/letta/interfaces/openai_streaming_interface.py
@@ -9,6 +9,7 @@ from openai.types.chat.chat_completion_chunk import ChatCompletionChunk
from letta.constants import DEFAULT_MESSAGE_TOOL, DEFAULT_MESSAGE_TOOL_KWARG
from letta.helpers.datetime_helpers import get_utc_timestamp_ns, ns_to_ms
from letta.llm_api.openai_client import is_openai_reasoning_model
+from letta.local_llm.utils import num_tokens_from_functions, num_tokens_from_messages
from letta.log import get_logger
from letta.otel.context import get_ctx_attributes
from letta.otel.metric_registry import MetricRegistry
@@ -19,6 +20,7 @@ from letta.schemas.message import Message
from letta.schemas.openai.chat_completion_response import FunctionCall, ToolCall
from letta.server.rest_api.json_parser import OptimisticJSONParser
from letta.streaming_utils import JSONInnerThoughtsExtractor
+from letta.utils import count_tokens
logger = get_logger(__name__)
@@ -30,7 +32,14 @@ class OpenAIStreamingInterface:
and detection of tool call events.
"""
- def __init__(self, use_assistant_message: bool = False, put_inner_thoughts_in_kwarg: bool = False):
+ def __init__(
+ self,
+ use_assistant_message: bool = False,
+ put_inner_thoughts_in_kwarg: bool = False,
+ is_openai_proxy: bool = False,
+ messages: Optional[list] = None,
+ tools: Optional[list] = None,
+ ):
self.use_assistant_message = use_assistant_message
self.assistant_message_tool_name = DEFAULT_MESSAGE_TOOL
self.assistant_message_tool_kwarg = DEFAULT_MESSAGE_TOOL_KWARG
@@ -53,10 +62,19 @@ class OpenAIStreamingInterface:
self.message_id = None
self.model = None
- # token counters
+ # Token counters (from OpenAI usage)
self.input_tokens = 0
self.output_tokens = 0
+ # Fallback token counters (using tiktoken cl200k-base)
+ self.fallback_input_tokens = 0
+ self.fallback_output_tokens = 0
+
+ # Store messages and tools for fallback counting
+ self.is_openai_proxy = is_openai_proxy
+ self.messages = messages or []
+ self.tools = tools or []
+
self.content_buffer: list[str] = []
self.tool_call_name: str | None = None
self.tool_call_id: str | None = None
@@ -95,6 +113,18 @@ class OpenAIStreamingInterface:
Iterates over the OpenAI stream, yielding SSE events.
It also collects tokens and detects if a tool call is triggered.
"""
+ # Fallback input token counting - this should only be required for non-OpenAI providers using the OpenAI client (e.g. LMStudio)
+ if self.is_openai_proxy:
+ if self.messages:
+ # Convert messages to dict format for token counting
+ message_dicts = [msg.to_openai_dict() if hasattr(msg, "to_openai_dict") else msg for msg in self.messages]
+ self.fallback_input_tokens = num_tokens_from_messages(message_dicts) # fallback to gpt-4 cl100k-base
+
+ if self.tools:
+ # Convert tools to dict format for token counting
+ tool_dicts = [tool["function"] if isinstance(tool, dict) and "function" in tool else tool for tool in self.tools]
+ self.fallback_input_tokens += num_tokens_from_functions(tool_dicts)
+
first_chunk = True
try:
async with stream:
@@ -113,6 +143,9 @@ class OpenAIStreamingInterface:
metric_attributes["model.name"] = chunk.model
MetricRegistry().ttft_ms_histogram.record(ns_to_ms(ttft_ns), metric_attributes)
+ if self.is_openai_proxy:
+ self.fallback_output_tokens += count_tokens(chunk.model_dump_json())
+
first_chunk = False
if not self.model or not self.message_id:
@@ -153,6 +186,9 @@ class OpenAIStreamingInterface:
tool_call.function.arguments
)
+ if self.is_openai_proxy:
+ self.fallback_output_tokens += count_tokens(tool_call.function.arguments)
+
# If we have inner thoughts, we should output them as a chunk
if updates_inner_thoughts:
if prev_message_type and prev_message_type != "reasoning_message":
diff --git a/tests/configs/llm_model_configs/bartowski-ministral-8b-instruct-2410.json b/tests/configs/llm_model_configs/bartowski-ministral-8b-instruct-2410.json
new file mode 100644
index 000000000..c2baa66ac
--- /dev/null
+++ b/tests/configs/llm_model_configs/bartowski-ministral-8b-instruct-2410.json
@@ -0,0 +1,8 @@
+{
+ "context_window": 8192,
+ "model": "ministral-8b-instruct-2410",
+ "model_endpoint_type": "openai",
+ "model_endpoint": "http://localhost:1234/v1",
+ "model_wrapper": null,
+ "provider_name": "lmstudio_openai"
+}
diff --git a/tests/configs/llm_model_configs/lmstudio-meta-llama-3.1-8b-instruct.json b/tests/configs/llm_model_configs/lmstudio-meta-llama-3.1-8b-instruct.json
new file mode 100644
index 000000000..fdf80602e
--- /dev/null
+++ b/tests/configs/llm_model_configs/lmstudio-meta-llama-3.1-8b-instruct.json
@@ -0,0 +1,8 @@
+{
+ "context_window": 8192,
+ "model": "meta-llama-3.1-8b-instruct",
+ "model_endpoint_type": "openai",
+ "model_endpoint": "http://localhost:1234/v1",
+ "model_wrapper": null,
+ "provider_name": "lmstudio_openai"
+}
diff --git a/tests/configs/llm_model_configs/lmstudio-qwen-2.5-7b-instruct.json b/tests/configs/llm_model_configs/lmstudio-qwen-2.5-7b-instruct.json
new file mode 100644
index 000000000..a737f1a55
--- /dev/null
+++ b/tests/configs/llm_model_configs/lmstudio-qwen-2.5-7b-instruct.json
@@ -0,0 +1,8 @@
+{
+ "context_window": 8192,
+ "model": "qwen2.5-7b-instruct",
+ "model_endpoint_type": "openai",
+ "model_endpoint": "http://localhost:1234/v1",
+ "model_wrapper": null,
+ "provider_name": "lmstudio_openai"
+}
diff --git a/tests/configs/llm_model_configs/mlx-meta-llama-3.1-8b-instruct-8bit.json b/tests/configs/llm_model_configs/mlx-meta-llama-3.1-8b-instruct-8bit.json
new file mode 100644
index 000000000..f6be77495
--- /dev/null
+++ b/tests/configs/llm_model_configs/mlx-meta-llama-3.1-8b-instruct-8bit.json
@@ -0,0 +1,8 @@
+{
+ "context_window": 8192,
+ "model": "meta-llama-3.1-8b-instruct-8bit",
+ "model_endpoint_type": "openai",
+ "model_endpoint": "http://localhost:1234/v1",
+ "model_wrapper": null,
+ "provider_name": "lmstudio_openai"
+}
diff --git a/tests/configs/llm_model_configs/mlx-ministral-8b-instruct-2410.json b/tests/configs/llm_model_configs/mlx-ministral-8b-instruct-2410.json
new file mode 100644
index 000000000..c2baa66ac
--- /dev/null
+++ b/tests/configs/llm_model_configs/mlx-ministral-8b-instruct-2410.json
@@ -0,0 +1,8 @@
+{
+ "context_window": 8192,
+ "model": "ministral-8b-instruct-2410",
+ "model_endpoint_type": "openai",
+ "model_endpoint": "http://localhost:1234/v1",
+ "model_wrapper": null,
+ "provider_name": "lmstudio_openai"
+}
diff --git a/tests/configs/llm_model_configs/mlx-qwen-2.5-7b-instruct.json b/tests/configs/llm_model_configs/mlx-qwen-2.5-7b-instruct.json
new file mode 100644
index 000000000..50b15d56c
--- /dev/null
+++ b/tests/configs/llm_model_configs/mlx-qwen-2.5-7b-instruct.json
@@ -0,0 +1,8 @@
+{
+ "context_window": 8192,
+ "model": "qwen-2.5-7b-instruct",
+ "model_endpoint_type": "openai",
+ "model_endpoint": "http://localhost:1234/v1",
+ "model_wrapper": null,
+ "provider_name": "lmstudio_openai"
+}
diff --git a/tests/integration_test_send_message.py b/tests/integration_test_send_message.py
index 314cf702f..2952e2fda 100644
--- a/tests/integration_test_send_message.py
+++ b/tests/integration_test_send_message.py
@@ -131,6 +131,24 @@ reasoning_configs = [
"openai-o1.json",
"openai-o3.json",
"openai-o4-mini.json",
+ # "azure-gpt-4o-mini.json",
+ # "claude-4-sonnet.json",
+ # "claude-3-5-sonnet.json",
+ # "claude-3-7-sonnet.json",
+ # "claude-3-7-sonnet-extended.json",
+ # "bedrock-claude-4-sonnet.json",
+ # "gemini-1.5-pro.json",
+ # "gemini-2.5-flash-vertex.json",
+ # "gemini-2.5-pro-vertex.json",
+ # "together-qwen-2.5-72b-instruct.json",
+ # "ollama.json", # TODO (cliandy): enable this in ollama testing
+ # TODO @jnjpng: not supported in CI yet, uncomment to test locally (requires lmstudio running locally with respective models loaded)
+ # "lmstudio-meta-llama-3.1-8b-instruct.json",
+ # "lmstudio-qwen-2.5-7b-instruct.json",
+ # "mlx-qwen-2.5-7b-instruct.json",
+ # "mlx-meta-llama-3.1-8b-instruct-8bit.json",
+ # "mlx-ministral-8b-instruct-2410.json",
+ # "bartowski-ministral-8b-instruct-2410.json"
]