Skip to content

armgit / zed

Public Imported from GitHub
0

Code at the speed of thought – Zed is a high-performance, multiplayer code editor from the creators of Atom and Tree-sitter.

lmstudio: Fix context wheel by including token usage in streaming responses (#57861)

Gabriele Ancillai04d03d1fe5af
diff --git a/crates/language_models/src/provider/lmstudio.rs b/crates/language_models/src/provider/lmstudio.rs
index 03b9451726a..518b4a26155 100644
--- a/crates/language_models/src/provider/lmstudio.rs
+++ b/crates/language_models/src/provider/lmstudio.rs
@@ -427,6 +427,9 @@ impl LmStudioLanguageModel {
model: self.model.name.clone(),
messages,
stream: true,
+ stream_options: Some(lmstudio::StreamOptions {
+ include_usage: true,
+ }),
max_tokens: Some(-1),
stop: Some(request.stop),
// In LM Studio you can configure specific settings you'd like to use for your model.
@@ -591,13 +594,23 @@ impl LmStudioEventMapper {
&mut self,
event: lmstudio::ResponseStreamEvent,
) -> Vec<Result<LanguageModelCompletionEvent, LanguageModelCompletionError>> {
+ let mut events = Vec::new();
+
+ if let Some(usage) = event.usage {
+ events.push(Ok(LanguageModelCompletionEvent::UsageUpdate(TokenUsage {
+ input_tokens: usage.prompt_tokens,
+ output_tokens: usage.completion_tokens,
+ cache_creation_input_tokens: 0,
+ cache_read_input_tokens: 0,
+ })));
+ }
+
+ // The final usage summary chunk from OpenAI-compatible servers has an empty choices array.
+ // Return accumulated events instead of treating it as an error.
let Some(choice) = event.choices.into_iter().next() else {
- return vec![Err(LanguageModelCompletionError::from(anyhow!(
- "Response contained no choices"
- )))];
+ return events;
};
- let mut events = Vec::new();
if let Some(content) = choice.delta.content {
events.push(Ok(LanguageModelCompletionEvent::Text(content)));
}
@@ -636,15 +649,6 @@ impl LmStudioEventMapper {
}
}
- if let Some(usage) = event.usage {
- events.push(Ok(LanguageModelCompletionEvent::UsageUpdate(TokenUsage {
- input_tokens: usage.prompt_tokens,
- output_tokens: usage.completion_tokens,
- cache_creation_input_tokens: 0,
- cache_read_input_tokens: 0,
- })));
- }
-
match choice.finish_reason.as_deref() {
Some("stop") => {
events.push(Ok(LanguageModelCompletionEvent::Stop(StopReason::EndTurn)));
@@ -691,6 +695,142 @@ struct RawToolCall {
arguments: String,
}
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use lmstudio::{ChoiceDelta, ResponseMessageDelta, ResponseStreamEvent, Usage};
+
+ fn make_event(choices: Vec<ChoiceDelta>, usage: Option<Usage>) -> ResponseStreamEvent {
+ ResponseStreamEvent {
+ created: 0,
+ model: "test-model".to_string(),
+ object: "chat.completion.chunk".to_string(),
+ choices,
+ usage,
+ }
+ }
+
+ fn make_content_choice(content: &str) -> ChoiceDelta {
+ ChoiceDelta {
+ index: 0,
+ delta: ResponseMessageDelta {
+ role: None,
+ content: Some(content.to_string()),
+ reasoning_content: None,
+ tool_calls: None,
+ },
+ finish_reason: None,
+ }
+ }
+
+ fn make_stop_choice() -> ChoiceDelta {
+ ChoiceDelta {
+ index: 0,
+ delta: ResponseMessageDelta {
+ role: None,
+ content: None,
+ reasoning_content: None,
+ tool_calls: None,
+ },
+ finish_reason: Some("stop".to_string()),
+ }
+ }
+
+ // OpenAI-compatible servers send a final chunk with usage data and an empty
+ // choices array. Before this fix, the mapper returned an error for empty
+ // choices, discarding usage entirely.
+ #[test]
+ fn test_usage_in_final_empty_choices_chunk() {
+ let mut mapper = LmStudioEventMapper::new();
+ let event = make_event(
+ vec![],
+ Some(Usage {
+ prompt_tokens: 10,
+ completion_tokens: 20,
+ total_tokens: 30,
+ }),
+ );
+
+ let results: Vec<_> = mapper
+ .map_event(event)
+ .into_iter()
+ .map(|r| r.unwrap())
+ .collect();
+
+ assert_eq!(
+ results,
+ vec![LanguageModelCompletionEvent::UsageUpdate(TokenUsage {
+ input_tokens: 10,
+ output_tokens: 20,
+ cache_creation_input_tokens: 0,
+ cache_read_input_tokens: 0,
+ })]
+ );
+ }
+
+ #[test]
+ fn test_empty_choices_without_usage_returns_empty() {
+ let mut mapper = LmStudioEventMapper::new();
+ let event = make_event(vec![], None);
+
+ let results = mapper.map_event(event);
+
+ assert!(results.is_empty());
+ }
+
+ // Usage data can also arrive in a regular chunk that also contains content.
+ // Both events must be emitted, with UsageUpdate first.
+ #[test]
+ fn test_usage_emitted_alongside_content() {
+ let mut mapper = LmStudioEventMapper::new();
+ let event = make_event(
+ vec![make_content_choice("Hello!")],
+ Some(Usage {
+ prompt_tokens: 5,
+ completion_tokens: 3,
+ total_tokens: 8,
+ }),
+ );
+
+ let results: Vec<_> = mapper
+ .map_event(event)
+ .into_iter()
+ .map(|r| r.unwrap())
+ .collect();
+
+ assert_eq!(
+ results[0],
+ LanguageModelCompletionEvent::UsageUpdate(TokenUsage {
+ input_tokens: 5,
+ output_tokens: 3,
+ cache_creation_input_tokens: 0,
+ cache_read_input_tokens: 0,
+ })
+ );
+ assert_eq!(
+ results[1],
+ LanguageModelCompletionEvent::Text("Hello!".to_string())
+ );
+ }
+
+ #[test]
+ fn test_stop_event_emitted_on_finish_reason() {
+ let mut mapper = LmStudioEventMapper::new();
+ let event = make_event(vec![make_stop_choice()], None);
+
+ let results: Vec<_> = mapper
+ .map_event(event)
+ .into_iter()
+ .map(|r| r.unwrap())
+ .collect();
+
+ assert_eq!(
+ results,
+ vec![LanguageModelCompletionEvent::Stop(StopReason::EndTurn)]
+ );
+ }
+}
+
fn add_message_content_part(
new_part: lmstudio::MessagePart,
role: Role,
diff --git a/crates/lmstudio/src/lmstudio.rs b/crates/lmstudio/src/lmstudio.rs
index 57963bbb040..4f5c977f129 100644
--- a/crates/lmstudio/src/lmstudio.rs
+++ b/crates/lmstudio/src/lmstudio.rs
@@ -207,12 +207,19 @@ pub struct FunctionContent {
pub arguments: String,
}
+#[derive(Serialize, Debug)]
+pub struct StreamOptions {
+ pub include_usage: bool,
+}
+
#[derive(Serialize, Debug)]
pub struct ChatCompletionRequest {
pub model: String,
pub messages: Vec<ChatMessage>,
pub stream: bool,
#[serde(skip_serializing_if = "Option::is_none")]
+ pub stream_options: Option<StreamOptions>,
+ #[serde(skip_serializing_if = "Option::is_none")]
pub max_tokens: Option<i32>,
#[serde(skip_serializing_if = "Option::is_none")]
pub stop: Option<Vec<String>>,