Code at the speed of thought – Zed is a high-performance, multiplayer code editor from the creators of Atom and Tree-sitter.
lmstudio: Fix context wheel by including token usage in streaming responses (#57861)
Gabriele Ancillai04d03d1fe5af
diff --git a/crates/language_models/src/provider/lmstudio.rs b/crates/language_models/src/provider/lmstudio.rsindex 03b9451726a..518b4a26155 100644--- a/crates/language_models/src/provider/lmstudio.rs+++ b/crates/language_models/src/provider/lmstudio.rs@@ -427,6 +427,9 @@ impl LmStudioLanguageModel {model: self.model.name.clone(),messages,stream: true,+ stream_options: Some(lmstudio::StreamOptions {+ include_usage: true,+ }),max_tokens: Some(-1),stop: Some(request.stop),// In LM Studio you can configure specific settings you'd like to use for your model.@@ -591,13 +594,23 @@ impl LmStudioEventMapper {&mut self,event: lmstudio::ResponseStreamEvent,) -> Vec<Result<LanguageModelCompletionEvent, LanguageModelCompletionError>> {+ let mut events = Vec::new();++ if let Some(usage) = event.usage {+ events.push(Ok(LanguageModelCompletionEvent::UsageUpdate(TokenUsage {+ input_tokens: usage.prompt_tokens,+ output_tokens: usage.completion_tokens,+ cache_creation_input_tokens: 0,+ cache_read_input_tokens: 0,+ })));+ }++ // The final usage summary chunk from OpenAI-compatible servers has an empty choices array.+ // Return accumulated events instead of treating it as an error.let Some(choice) = event.choices.into_iter().next() else {- return vec![Err(LanguageModelCompletionError::from(anyhow!(- "Response contained no choices"- )))];+ return events;};- let mut events = Vec::new();if let Some(content) = choice.delta.content {events.push(Ok(LanguageModelCompletionEvent::Text(content)));}@@ -636,15 +649,6 @@ impl LmStudioEventMapper {}}- if let Some(usage) = event.usage {- events.push(Ok(LanguageModelCompletionEvent::UsageUpdate(TokenUsage {- input_tokens: usage.prompt_tokens,- output_tokens: usage.completion_tokens,- cache_creation_input_tokens: 0,- cache_read_input_tokens: 0,- })));- }-match choice.finish_reason.as_deref() {Some("stop") => {events.push(Ok(LanguageModelCompletionEvent::Stop(StopReason::EndTurn)));@@ -691,6 +695,142 @@ struct RawToolCall {arguments: String,}+#[cfg(test)]+mod tests {+ use super::*;+ use lmstudio::{ChoiceDelta, ResponseMessageDelta, ResponseStreamEvent, Usage};++ fn make_event(choices: Vec<ChoiceDelta>, usage: Option<Usage>) -> ResponseStreamEvent {+ ResponseStreamEvent {+ created: 0,+ model: "test-model".to_string(),+ object: "chat.completion.chunk".to_string(),+ choices,+ usage,+ }+ }++ fn make_content_choice(content: &str) -> ChoiceDelta {+ ChoiceDelta {+ index: 0,+ delta: ResponseMessageDelta {+ role: None,+ content: Some(content.to_string()),+ reasoning_content: None,+ tool_calls: None,+ },+ finish_reason: None,+ }+ }++ fn make_stop_choice() -> ChoiceDelta {+ ChoiceDelta {+ index: 0,+ delta: ResponseMessageDelta {+ role: None,+ content: None,+ reasoning_content: None,+ tool_calls: None,+ },+ finish_reason: Some("stop".to_string()),+ }+ }++ // OpenAI-compatible servers send a final chunk with usage data and an empty+ // choices array. Before this fix, the mapper returned an error for empty+ // choices, discarding usage entirely.+ #[test]+ fn test_usage_in_final_empty_choices_chunk() {+ let mut mapper = LmStudioEventMapper::new();+ let event = make_event(+ vec![],+ Some(Usage {+ prompt_tokens: 10,+ completion_tokens: 20,+ total_tokens: 30,+ }),+ );++ let results: Vec<_> = mapper+ .map_event(event)+ .into_iter()+ .map(|r| r.unwrap())+ .collect();++ assert_eq!(+ results,+ vec![LanguageModelCompletionEvent::UsageUpdate(TokenUsage {+ input_tokens: 10,+ output_tokens: 20,+ cache_creation_input_tokens: 0,+ cache_read_input_tokens: 0,+ })]+ );+ }++ #[test]+ fn test_empty_choices_without_usage_returns_empty() {+ let mut mapper = LmStudioEventMapper::new();+ let event = make_event(vec![], None);++ let results = mapper.map_event(event);++ assert!(results.is_empty());+ }++ // Usage data can also arrive in a regular chunk that also contains content.+ // Both events must be emitted, with UsageUpdate first.+ #[test]+ fn test_usage_emitted_alongside_content() {+ let mut mapper = LmStudioEventMapper::new();+ let event = make_event(+ vec![make_content_choice("Hello!")],+ Some(Usage {+ prompt_tokens: 5,+ completion_tokens: 3,+ total_tokens: 8,+ }),+ );++ let results: Vec<_> = mapper+ .map_event(event)+ .into_iter()+ .map(|r| r.unwrap())+ .collect();++ assert_eq!(+ results[0],+ LanguageModelCompletionEvent::UsageUpdate(TokenUsage {+ input_tokens: 5,+ output_tokens: 3,+ cache_creation_input_tokens: 0,+ cache_read_input_tokens: 0,+ })+ );+ assert_eq!(+ results[1],+ LanguageModelCompletionEvent::Text("Hello!".to_string())+ );+ }++ #[test]+ fn test_stop_event_emitted_on_finish_reason() {+ let mut mapper = LmStudioEventMapper::new();+ let event = make_event(vec![make_stop_choice()], None);++ let results: Vec<_> = mapper+ .map_event(event)+ .into_iter()+ .map(|r| r.unwrap())+ .collect();++ assert_eq!(+ results,+ vec![LanguageModelCompletionEvent::Stop(StopReason::EndTurn)]+ );+ }+}+fn add_message_content_part(new_part: lmstudio::MessagePart,role: Role,diff --git a/crates/lmstudio/src/lmstudio.rs b/crates/lmstudio/src/lmstudio.rsindex 57963bbb040..4f5c977f129 100644--- a/crates/lmstudio/src/lmstudio.rs+++ b/crates/lmstudio/src/lmstudio.rs@@ -207,12 +207,19 @@ pub struct FunctionContent {pub arguments: String,}+#[derive(Serialize, Debug)]+pub struct StreamOptions {+ pub include_usage: bool,+}+#[derive(Serialize, Debug)]pub struct ChatCompletionRequest {pub model: String,pub messages: Vec<ChatMessage>,pub stream: bool,#[serde(skip_serializing_if = "Option::is_none")]+ pub stream_options: Option<StreamOptions>,+ #[serde(skip_serializing_if = "Option::is_none")]pub max_tokens: Option<i32>,#[serde(skip_serializing_if = "Option::is_none")]pub stop: Option<Vec<String>>,