Code at the speed of thought – Zed is a high-performance, multiplayer code editor from the creators of Atom and Tree-sitter.
Add per-request model output limits (#64047)
MartinYe123459d996d8dbae
diff --git a/crates/agent/src/thread.rs b/crates/agent/src/thread.rsindex a1a5409ee1c..39f99aac0ca 100644--- a/crates/agent/src/thread.rs+++ b/crates/agent/src/thread.rs@@ -4150,6 +4150,7 @@ impl Thread {thinking_effort: self.thinking_effort.clone(),speed: self.speed(),compact_at_tokens: None,+ max_output_tokens: None,};log::debug!("Completion request built successfully");diff --git a/crates/agent_ui/src/buffer_codegen.rs b/crates/agent_ui/src/buffer_codegen.rsindex bff276966c0..e0d3b95277e 100644--- a/crates/agent_ui/src/buffer_codegen.rs+++ b/crates/agent_ui/src/buffer_codegen.rs@@ -557,6 +557,7 @@ impl CodegenAlternative {thinking_effort: None,speed: None,compact_at_tokens: None,+ max_output_tokens: None,}}))}@@ -639,6 +640,7 @@ impl CodegenAlternative {thinking_effort: None,speed: None,compact_at_tokens: None,+ max_output_tokens: None,}}))}diff --git a/crates/agent_ui/src/terminal_inline_assistant.rs b/crates/agent_ui/src/terminal_inline_assistant.rsindex 800a68a3f3c..efbd1d24949 100644--- a/crates/agent_ui/src/terminal_inline_assistant.rs+++ b/crates/agent_ui/src/terminal_inline_assistant.rs@@ -276,6 +276,7 @@ impl TerminalInlineAssistant {thinking_effort: None,speed: None,compact_at_tokens: None,+ max_output_tokens: None,}}))}diff --git a/crates/anthropic/src/completion.rs b/crates/anthropic/src/completion.rsindex d2a601ad55c..ed909521ae5 100644--- a/crates/anthropic/src/completion.rs+++ b/crates/anthropic/src/completion.rs@@ -290,6 +290,9 @@ pub fn into_anthropic(cache_mode: AnthropicPromptCacheMode,compaction_state_owner: &LanguageModelProviderId,) -> Result<crate::Request> {+ let max_output_tokens = request+ .max_output_tokens+ .map_or(max_output_tokens, |limit| limit.min(max_output_tokens));let mut new_messages: Vec<Message> = Vec::new();let mut system_message = String::new();let mut any_message_wants_cache = false;@@ -907,6 +910,7 @@ mod tests {thinking_effort: None,speed: None,compact_at_tokens: None,+ max_output_tokens: Some(1024),};let anthropic_request = into_anthropic(@@ -920,6 +924,10 @@ mod tests {).unwrap();+ assert_eq!(+ serde_json::to_value(&anthropic_request).unwrap()["max_tokens"],+ 1024+ );// No message content block should carry cache_control anymore; the// conversation breakpoint is set via top-level automatic caching.assert_eq!(anthropic_request.messages.len(), 1);@@ -1015,6 +1023,7 @@ mod tests {thinking_effort: None,speed: None,compact_at_tokens: None,+ max_output_tokens: Some(8192),};let anthropic_request = into_anthropic(@@ -1028,6 +1037,10 @@ mod tests {).unwrap();+ assert_eq!(+ serde_json::to_value(&anthropic_request).unwrap()["max_tokens"],+ 4096+ );assert!(anthropic_request.cache_control.is_none());assert!(matches!(anthropic_request.system,@@ -1075,6 +1088,7 @@ mod tests {thinking_effort: Some("xhigh".into()),speed: None,compact_at_tokens: None,+ max_output_tokens: None,};let anthropic_request = into_anthropic(@@ -1126,6 +1140,7 @@ mod tests {thinking_effort: None,speed: None,compact_at_tokens: None,+ max_output_tokens: None,};let anthropic_request = into_anthropic(@@ -1195,6 +1210,7 @@ mod tests {thinking_effort: None,speed: None,compact_at_tokens: None,+ max_output_tokens: None,};let anthropic_request = into_anthropic(@@ -1261,6 +1277,7 @@ mod tests {thinking_effort: None,speed: None,compact_at_tokens: None,+ max_output_tokens: None,};let anthropic_request = into_anthropic(@@ -1274,6 +1291,10 @@ mod tests {).unwrap();+ assert_eq!(+ serde_json::to_value(&anthropic_request).unwrap()["max_tokens"],+ 4096+ );assert!(anthropic_request.cache_control.is_none());assert!(matches!(anthropic_request.system,@@ -1301,6 +1322,7 @@ mod tests {thinking_allowed: true,speed: None,compact_at_tokens: None,+ max_output_tokens: None,};request.messages.push(LanguageModelRequestMessage {role: Role::Assistant,diff --git a/crates/copilot_chat/src/copilot_chat.rs b/crates/copilot_chat/src/copilot_chat.rsindex 92073f5c0ba..71ef0895d96 100644--- a/crates/copilot_chat/src/copilot_chat.rs+++ b/crates/copilot_chat/src/copilot_chat.rs@@ -279,8 +279,9 @@ impl Model {self.capabilities.limits.max_context_window_tokens as u64}- pub fn max_output_tokens(&self) -> usize {- self.capabilities.limits.max_output_tokens+ pub fn max_output_tokens(&self) -> Option<u64> {+ let limit = self.capabilities.limits.max_output_tokens as u64;+ (limit > 0).then_some(limit)}pub fn supports_tools(&self) -> bool {@@ -356,6 +357,8 @@ pub struct Request {pub temperature: f32,pub model: String,pub messages: Vec<ChatMessage>,+ #[serde(default, skip_serializing_if = "Option::is_none")]+ pub max_tokens: Option<u64>,#[serde(default, skip_serializing_if = "Vec::is_empty")]pub tools: Vec<Tool>,#[serde(default, skip_serializing_if = "Option::is_none")]diff --git a/crates/copilot_chat/src/model.rs b/crates/copilot_chat/src/model.rsindex bbd12c4d5aa..12d524c318a 100644--- a/crates/copilot_chat/src/model.rs+++ b/crates/copilot_chat/src/model.rs@@ -161,6 +161,12 @@ impl LanguageModel for CopilotChatLanguageModel {let copilot_chat = self.copilot_chat.clone();let request_limiter = self.request_limiter.clone();let future = cx.spawn(async move |cx| {+ let max_output_tokens = model+ .max_output_tokens()+ .or(request.max_output_tokens)+ .ok_or_else(|| {+ anyhow!("Copilot did not provide an output limit for this model")+ })?;let effort = request.thinking_effort.as_ref()@@ -170,7 +176,7 @@ impl LanguageModel for CopilotChatLanguageModel {request,model.id().to_string(),0.0,- model.max_output_tokens() as u64,+ max_output_tokens,if model.supports_adaptive_thinking() {AnthropicModelMode::Thinking {budget_tokens: None,@@ -180,7 +186,7 @@ impl LanguageModel for CopilotChatLanguageModel {budget_tokens: compute_thinking_budget(model.min_thinking_budget(),model.max_thinking_budget(),- model.max_output_tokens() as u32,+ max_output_tokens.min(u32::MAX as u64) as u32,),}} else {@@ -814,6 +820,9 @@ fn into_copilot_chat(model: &CopilotChatModel,request: LanguageModelRequest,) -> Result<CopilotChatRequest> {+ let max_tokens = request+ .max_output_tokens+ .and_then(|_| request.effective_max_output_tokens(model.max_output_tokens()));let temperature = request.temperature;let tool_choice = request.tool_choice;let thinking_allowed = request.thinking_allowed;@@ -1008,6 +1017,7 @@ fn into_copilot_chat(temperature: temperature.unwrap_or(0.1),model: model.id().to_string(),messages,+ max_tokens,tools,tool_choice: tool_choice.map(|choice| match choice {LanguageModelToolChoice::Auto => ToolChoice::Auto,@@ -1015,11 +1025,13 @@ fn into_copilot_chat(LanguageModelToolChoice::None => ToolChoice::None,}),thinking_budget: if thinking_allowed && model.supports_thinking() {- compute_thinking_budget(- model.min_thinking_budget(),- model.max_thinking_budget(),- model.max_output_tokens() as u32,- )+ model.max_output_tokens().or(max_tokens).and_then(|limit| {+ compute_thinking_budget(+ model.min_thinking_budget(),+ model.max_thinking_budget(),+ limit.min(u32::MAX as u64) as u32,+ )+ })} else {None},@@ -1064,6 +1076,9 @@ fn into_copilot_responses() -> Result<copilot_responses::Request> {use copilot_responses as responses;+ let max_output_tokens = request+ .max_output_tokens+ .and_then(|_| request.effective_max_output_tokens(model.max_output_tokens()));let LanguageModelRequest {thread_id: _,prompt_id: _,@@ -1077,6 +1092,7 @@ fn into_copilot_responses(thinking_effort,speed: _,compact_at_tokens: _,+ max_output_tokens: _,} = request;let mut input_items: Vec<responses::ResponseInputItem> = Vec::new();@@ -1255,6 +1271,7 @@ fn into_copilot_responses(input: input_items,stream: model.uses_streaming(),temperature,+ max_output_tokens,tools: converted_tools,tool_choice: mapped_tool_choice,reasoning: if thinking_allowed {@@ -1523,6 +1540,54 @@ mod tests {);}+ #[test]+ fn request_output_limits_reach_copilot_payloads() -> Result<()> {+ let model = test_responses_model();+ for (limit, expected) in [+ (None, None),+ (Some(1024), Some(1024)),+ (Some(u64::MAX), Some(4096)),+ ] {+ let request = LanguageModelRequest {+ max_output_tokens: limit,+ ..Default::default()+ };+ let responses = serde_json::to_value(into_copilot_responses(&model, request.clone())?)?;+ let chat = serde_json::to_value(into_copilot_chat(&model, request)?)?;+ assert_eq!(+ responses.get("max_output_tokens").cloned(),+ expected.map(|value| json!(value))+ );+ assert_eq!(+ chat.get("max_tokens").cloned(),+ expected.map(|value| json!(value))+ );+ }+ let mut value = serde_json::to_value(test_responses_model())?;+ value["capabilities"]["limits"]+ .as_object_mut()+ .unwrap()+ .remove("max_output_tokens");+ let model = serde_json::from_value(value)?;+ for limit in [None, Some(1024)] {+ let request = LanguageModelRequest {+ max_output_tokens: limit,+ ..Default::default()+ };+ let responses = serde_json::to_value(into_copilot_responses(&model, request.clone())?)?;+ let chat = serde_json::to_value(into_copilot_chat(&model, request)?)?;+ assert_eq!(+ responses.get("max_output_tokens").cloned(),+ limit.map(|value| json!(value))+ );+ assert_eq!(+ chat.get("max_tokens").cloned(),+ limit.map(|value| json!(value))+ );+ }+ Ok(())+ }+#[test]fn into_copilot_responses_replays_reasoning_details() {let model = test_responses_model();diff --git a/crates/copilot_chat/src/responses.rs b/crates/copilot_chat/src/responses.rsindex 5cb8cf0af4a..597b058146e 100644--- a/crates/copilot_chat/src/responses.rs+++ b/crates/copilot_chat/src/responses.rs@@ -16,6 +16,8 @@ pub struct Request {pub stream: bool,#[serde(skip_serializing_if = "Option::is_none")]pub temperature: Option<f32>,+ #[serde(skip_serializing_if = "Option::is_none")]+ pub max_output_tokens: Option<u64>,#[serde(skip_serializing_if = "Vec::is_empty")]pub tools: Vec<ToolDefinition>,#[serde(skip_serializing_if = "Option::is_none")]diff --git a/crates/git_ui/src/git_panel.rs b/crates/git_ui/src/git_panel.rsindex 7f10a4a8829..26126aeaa91 100644--- a/crates/git_ui/src/git_panel.rs+++ b/crates/git_ui/src/git_panel.rs@@ -4166,6 +4166,7 @@ impl GitPanel {thinking_effort: None,speed: None,compact_at_tokens: None,+ max_output_tokens: None,};let stream = model.stream_completion_text(request, cx);diff --git a/crates/google_ai/src/completion.rs b/crates/google_ai/src/completion.rsindex 60ee81751b5..fda7eb16354 100644--- a/crates/google_ai/src/completion.rs+++ b/crates/google_ai/src/completion.rs@@ -113,6 +113,10 @@ pub fn into_google(}let thinking_config = thinking_config_for_request(&request, &model_id, mode);+ let max_output_tokens = request+ .effective_max_output_tokens(None)+ .map(usize::try_from)+ .transpose()?;let system_instructions = if request.messages@@ -179,7 +183,7 @@ pub fn into_google(generation_config: Some(GenerationConfig {candidate_count: Some(1),stop_sequences: Some(request.stop),- max_output_tokens: None,+ max_output_tokens,temperature: request.temperature.map(|t| t as f64),thinking_config,top_p: None,@@ -516,6 +520,11 @@ mod tests {).unwrap();+ assert!(+ serde_json::to_value(&request).unwrap()["generationConfig"]+ .get("maxOutputTokens")+ .is_none()+ );let thinking_config = request.generation_config.unwrap().thinking_config.unwrap();assert_eq!(thinking_config.include_thoughts, Some(true));assert_eq!(thinking_config.thinking_level, Some(ThinkingLevel::Low));@@ -542,6 +551,7 @@ mod tests {}});let mut request = text_request();+ request.max_output_tokens = Some(1024);request.tools = vec![LanguageModelRequestTool::function("grep".to_string(),"Search files".to_string(),@@ -556,6 +566,7 @@ mod tests {).unwrap();let serialized = serde_json::to_value(request).unwrap();+ assert_eq!(serialized["generationConfig"]["maxOutputTokens"], 1024);let declaration = &serialized["tools"][0]["functionDeclarations"][0];assert_eq!(declaration["parametersJsonSchema"], input_schema);@@ -566,6 +577,7 @@ mod tests {fn into_google_turns_off_budget_thinking_when_supported() {let mut request = text_request();request.thinking_allowed = false;+ request.max_output_tokens = Some(0);let request = into_google(request,@@ -576,6 +588,10 @@ mod tests {).unwrap();+ assert_eq!(+ serde_json::to_value(&request).unwrap()["generationConfig"]["maxOutputTokens"],+ 0+ );let thinking_config = request.generation_config.unwrap().thinking_config.unwrap();assert_eq!(thinking_config.thinking_budget, Some(0));assert_eq!(thinking_config.include_thoughts, None);diff --git a/crates/language_model/src/language_model.rs b/crates/language_model/src/language_model.rsindex d4bb4d5f13d..38c89fd3e15 100644--- a/crates/language_model/src/language_model.rs+++ b/crates/language_model/src/language_model.rs@@ -171,6 +171,11 @@ pub trait LanguageModel: Send + Sync {false}+ /// Whether native compaction honors `LanguageModelRequest::max_output_tokens`.+ fn supports_explicit_compaction_output_limit(&self) -> bool {+ false+ }+/// The provider-enforced input size required for explicit compaction.fn minimum_explicit_compaction_input_tokens(&self) -> Option<u64> {None@@ -210,7 +215,18 @@ pub trait LanguageModel: Send + Sync {false}+ /// Returns the input token ceiling.fn max_token_count(&self) -> u64;++ /// Returns the combined input and output ceiling, if one applies.+ ///+ /// The conservative default shares the input ceiling with output. `None`+ /// means generation does not consume that window, not merely that the API+ /// validates input separately or stops generation at the window boundary.+ fn max_total_tokens(&self) -> Option<u64> {+ Some(self.max_token_count())+ }+fn max_output_tokens(&self) -> Option<u64> {None}diff --git a/crates/language_model_core/src/request.rs b/crates/language_model_core/src/request.rsindex 68222eff967..a6426235dfc 100644--- a/crates/language_model_core/src/request.rs+++ b/crates/language_model_core/src/request.rs@@ -473,9 +473,23 @@ pub struct LanguageModelRequest {pub speed: Option<Speed>,#[serde(default, skip_serializing_if = "Option::is_none")]pub compact_at_tokens: Option<u64>,+ /// An output-token ceiling, including reasoning tokens where the provider counts them.+ ///+ /// Providers must forward this limit (clamped to a known model maximum), or reject+ /// capped requests when unsupported. `None` preserves the provider's default.+ #[serde(default, skip_serializing_if = "Option::is_none")]+ pub max_output_tokens: Option<u64>,}impl LanguageModelRequest {+ /// Combines the request's output limit with the model's output limit.+ pub fn effective_max_output_tokens(&self, model_maximum: Option<u64>) -> Option<u64> {+ match (self.max_output_tokens, model_maximum) {+ (Some(request), Some(model)) => Some(request.min(model)),+ (request, model) => request.or(model),+ }+ }+pub fn contains_custom_tool_input(&self) -> bool {self.tools.iter()@@ -523,6 +537,29 @@ pub struct LanguageModelResponseMessage {mod tests {use super::*;+ #[test]+ fn request_output_limit_serialization() -> serde_json::Result<()> {+ let request = LanguageModelRequest::default();+ let mut serialized = serde_json::to_value(&request)?;+ assert!(serialized.get("max_output_tokens").is_none());+ assert_eq!(+ serde_json::from_value::<LanguageModelRequest>(serialized.clone())?,+ request+ );++ serialized["max_output_tokens"] = serde_json::json!(1024);+ let capped: LanguageModelRequest = serde_json::from_value(serialized.clone())?;+ assert_eq!(capped.max_output_tokens, Some(1024));+ assert_eq!(serde_json::to_value(capped)?, serialized);++ serialized["max_output_tokens"] = serde_json::Value::Null;+ assert_eq!(+ serde_json::from_value::<LanguageModelRequest>(serialized)?,+ request+ );+ Ok(())+ }+#[test]fn test_language_model_tool_result_content_deserialization() {// Test plain stringdiff --git a/crates/language_models/src/provider/anthropic.rs b/crates/language_models/src/provider/anthropic.rsindex 73b374302f6..686ff2b702f 100644--- a/crates/language_models/src/provider/anthropic.rs+++ b/crates/language_models/src/provider/anthropic.rs@@ -467,6 +467,7 @@ mod tests {let model = direct_anthropic_test_model(&provider);assert!(model.supports_explicit_compaction());+ assert_eq!(model.max_total_tokens(), Some(model.max_token_count()));assert_eq!(model.minimum_explicit_compaction_input_tokens(),Some(anthropic::MIN_COMPACTION_TRIGGER_TOKENS)@@ -570,6 +571,7 @@ mod tests {store_key.await.unwrap();let model = direct_anthropic_test_model(&provider);let request = LanguageModelRequest {+ max_output_tokens: Some(8192),messages: vec![LanguageModelRequestMessage {role: language_model::Role::User,content: vec![MessageContent::Text("Retain this context.".to_string())],@@ -614,6 +616,7 @@ mod tests {.is_some_and(|header| header.contains(anthropic::COMPACTION_BETA_HEADER)));let body = serde_json::from_str::<serde_json::Value>(&body).unwrap();+ assert_eq!(body["max_tokens"], 8192);assert_eq!(body["context_management"],json!({@@ -798,6 +801,10 @@ impl LanguageModel for AnthropicModel {self.model.supports_compaction}+ fn supports_explicit_compaction_output_limit(&self) -> bool {+ self.supports_explicit_compaction()+ }+fn minimum_explicit_compaction_input_tokens(&self) -> Option<u64> {self.supports_explicit_compaction().then_some(anthropic::MIN_COMPACTION_TRIGGER_TOKENS)diff --git a/crates/language_models/src/provider/bedrock.rs b/crates/language_models/src/provider/bedrock.rsindex d3b5c41efbe..092dcd1ae10 100644--- a/crates/language_models/src/provider/bedrock.rs+++ b/crates/language_models/src/provider/bedrock.rs@@ -2004,6 +2004,9 @@ pub fn into_bedrock(guardrail_identifier: Option<String>,guardrail_version: Option<String>,) -> Result<bedrock::Request> {+ let max_output_tokens = request+ .max_output_tokens+ .map_or(max_output_tokens, |limit| limit.min(max_output_tokens));if request.contains_custom_tool_input() {anyhow::bail!("Bedrock does not support custom tools");}@@ -2948,10 +2951,15 @@ mod tests {// Claude Opus 5 runs adaptive thinking by default when the `thinking`// field is omitted, so suppressing thinking requires an explicit// `disabled` opt-out. Earlier Claude models treat omission as "off".- for (model, expects_explicit_opt_out) in [- ("us.anthropic.claude-opus-5", true),- ("global.anthropic.claude-opus-5", true),- ("us.anthropic.claude-opus-4-8", false),+ for (model, expects_explicit_opt_out, output_limit, expected_output) in [+ ("us.anthropic.claude-opus-5", true, None, 128_000),+ ("global.anthropic.claude-opus-5", true, Some(8192), 8192),+ (+ "us.anthropic.claude-opus-4-8",+ false,+ Some(256_000),+ 128_000,+ ),] {let request = into_bedrock(LanguageModelRequest {@@ -2962,6 +2970,7 @@ mod tests {reasoning_details: None,}],thinking_allowed: false,+ max_output_tokens: output_limit,..Default::default()},model.to_string(),@@ -2977,6 +2986,7 @@ mod tests {).unwrap();+ assert_eq!(request.max_tokens, expected_output);if expects_explicit_opt_out {assert!(matches!(request.thinking, Some(bedrock::Thinking::Disabled)),diff --git a/crates/language_models/src/provider/deepseek.rs b/crates/language_models/src/provider/deepseek.rsindex b2365fbbc61..19a988adc2e 100644--- a/crates/language_models/src/provider/deepseek.rs+++ b/crates/language_models/src/provider/deepseek.rs@@ -366,6 +366,7 @@ pub fn into_deepseek(model: &deepseek::Model,max_output_tokens: Option<u64>,) -> Result<deepseek::Request> {+ let max_output_tokens = request.effective_max_output_tokens(max_output_tokens);if request.contains_custom_tool_input() {anyhow::bail!("DeepSeek does not support custom tools");}@@ -666,3 +667,24 @@ impl DeepSeekEventMapper {events}}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn request_output_limits_reach_deepseek_payloads() -> Result<()> {+ for (limit, expected) in [(None, 4096), (Some(1024), 1024), (Some(8192), 4096)] {+ let request = into_deepseek(+ LanguageModelRequest {+ max_output_tokens: limit,+ ..Default::default()+ },+ &deepseek::Model::V4Pro,+ Some(4096),+ )?;+ assert_eq!(serde_json::to_value(request)?["max_tokens"], expected);+ }+ Ok(())+ }+}diff --git a/crates/language_models/src/provider/google.rs b/crates/language_models/src/provider/google.rsindex f7a50f8ee94..57e00f8d021 100644--- a/crates/language_models/src/provider/google.rs+++ b/crates/language_models/src/provider/google.rs@@ -354,6 +354,11 @@ impl LanguageModel for GoogleLanguageModel {LanguageModelCompletionError,>,> {+ let mut request = request;+ if request.max_output_tokens.is_some() {+ request.max_output_tokens =+ request.effective_max_output_tokens(self.max_output_tokens());+ }let request = match into_google(request,self.model.request_id().to_string(),diff --git a/crates/language_models/src/provider/llama_cpp.rs b/crates/language_models/src/provider/llama_cpp.rsindex 3b18b732253..c21494ffc4f 100644--- a/crates/language_models/src/provider/llama_cpp.rs+++ b/crates/language_models/src/provider/llama_cpp.rs@@ -853,8 +853,8 @@ fn build_llama_cpp_request(model: model_name.to_string(),messages,stream: true,- // Let the server decide the output length (its `n_predict` default).- max_tokens: None,+ // Without an explicit limit, preserve the server's `n_predict` default.+ max_tokens: request.max_output_tokens.map(i32::try_from).transpose()?,stop: if request.stop.is_empty() {None} else {@@ -1702,6 +1702,7 @@ mod tests {supports_thinking: true,},LanguageModelRequest {+ max_output_tokens: Some(1024),messages: vec![language_model::LanguageModelRequestMessage {role: Role::Assistant,content: vec![@@ -1719,6 +1720,7 @@ mod tests {).unwrap();+ assert_eq!(serde_json::to_value(&request).unwrap()["max_tokens"], 1024);assert_eq!(request.messages.len(), 1);match &request.messages[0] {llama_cpp::ChatMessage::Assistant {@@ -1763,6 +1765,12 @@ mod tests {).unwrap();+ assert!(+ serde_json::to_value(&request)+ .unwrap()+ .get("max_tokens")+ .is_none()+ );assert_eq!(request.messages.len(), 1);match &request.messages[0] {llama_cpp::ChatMessage::Assistant {diff --git a/crates/language_models/src/provider/lmstudio.rs b/crates/language_models/src/provider/lmstudio.rsindex a2de082c1b4..310932cfc45 100644--- a/crates/language_models/src/provider/lmstudio.rs+++ b/crates/language_models/src/provider/lmstudio.rs@@ -425,7 +425,13 @@ impl LmStudioLanguageModel {stream_options: Some(lmstudio::StreamOptions {include_usage: true,}),- max_tokens: Some(-1),+ max_tokens: Some(+ request+ .max_output_tokens+ .map(i32::try_from)+ .transpose()?+ .unwrap_or(-1),+ ),stop: Some(request.stop),// In LM Studio you can configure specific settings you'd like to use for your model.// For example Qwen3 is recommended to be used with 0.7 temperature.diff --git a/crates/language_models/src/provider/mistral.rs b/crates/language_models/src/provider/mistral.rsindex d447cb12eee..0ebb4a1b698 100644--- a/crates/language_models/src/provider/mistral.rs+++ b/crates/language_models/src/provider/mistral.rs@@ -367,6 +367,7 @@ pub fn into_mistral(model: mistral::Model,max_output_tokens: Option<u64>,) -> Result<(mistral::Request, Option<String>)> {+ let max_output_tokens = request.effective_max_output_tokens(max_output_tokens);if request.contains_custom_tool_input() {anyhow::bail!("Mistral does not support custom tools");}@@ -850,7 +851,7 @@ mod tests {#[test]fn test_into_mistral_basic_conversion() {- let request = LanguageModelRequest {+ let request = |max_output_tokens| LanguageModelRequest {messages: vec![LanguageModelRequestMessage {role: Role::System,@@ -883,16 +884,32 @@ mod tests {thinking_effort: None,speed: Default::default(),compact_at_tokens: None,+ max_output_tokens,};- let (mistral_request, affinity) =- into_mistral(request, mistral::Model::MistralSmallLatest, None).unwrap();-- assert_eq!(mistral_request.model, "mistral-small-latest");- assert_eq!(mistral_request.temperature, Some(0.5));- assert_eq!(mistral_request.messages.len(), 2);- assert!(mistral_request.stream);- assert_eq!(affinity, Some("abcdef".into()));+ for (requested, maximum, expected) in [+ (None, None, None),+ (None, Some(4096), Some(4096)),+ (Some(1024), Some(4096), Some(1024)),+ (Some(8192), Some(4096), Some(4096)),+ (Some(1024), None, Some(1024)),+ ] {+ let (mistral_request, affinity) = into_mistral(+ request(requested),+ mistral::Model::MistralSmallLatest,+ maximum,+ )+ .unwrap();+ assert_eq!(mistral_request.model, "mistral-small-latest");+ assert_eq!(mistral_request.temperature, Some(0.5));+ assert_eq!(mistral_request.messages.len(), 2);+ assert!(mistral_request.stream);+ assert_eq!(affinity, Some("abcdef".into()));+ assert_eq!(+ serde_json::to_value(mistral_request).unwrap()["max_tokens"].as_u64(),+ expected+ );+ }}#[test]@@ -915,6 +932,7 @@ mod tests {thinking_effort: None,speed: Default::default(),compact_at_tokens: None,+ max_output_tokens: None,};let (mistral_request, _) =@@ -958,6 +976,7 @@ mod tests {thinking_effort: None,speed: None,compact_at_tokens: None,+ max_output_tokens: None,};let (mistral_request, _) =diff --git a/crates/language_models/src/provider/ollama.rs b/crates/language_models/src/provider/ollama.rsindex 9ef157f9655..6dc0175c1b1 100644--- a/crates/language_models/src/provider/ollama.rs+++ b/crates/language_models/src/provider/ollama.rs@@ -465,6 +465,7 @@ impl OllamaLanguageModel {stream: true,options: Some(ChatOptions {num_ctx: Some(self.model.max_tokens),+ num_predict: request.max_output_tokens.map(isize::try_from).transpose()?,// Only send stop tokens if explicitly provided. When empty/None,// Ollama will use the model's default stop tokens from its Modelfile.// Sending an empty array would override and disable the defaults.diff --git a/crates/language_models/src/provider/open_router.rs b/crates/language_models/src/provider/open_router.rsindex 84b4f7b898d..368ff34c3d9 100644--- a/crates/language_models/src/provider/open_router.rs+++ b/crates/language_models/src/provider/open_router.rs@@ -458,6 +458,7 @@ pub fn into_open_router(model: &Model,max_output_tokens: Option<u64>,) -> Result<open_router::Request> {+ let max_output_tokens = request.effective_max_output_tokens(max_output_tokens);if request.contains_custom_tool_input() {anyhow::bail!("OpenRouter does not support custom tools");}@@ -790,8 +791,9 @@ mod tests {None,);let thread_id = "internal-thread-id";- let request = LanguageModelRequest {+ let request = |max_output_tokens| LanguageModelRequest {thread_id: Some(thread_id.to_string()),+ max_output_tokens,messages: vec![language_model::LanguageModelRequestMessage {role: Role::User,content: vec![MessageContent::Text("Hello".to_string())],@@ -801,17 +803,27 @@ mod tests {..Default::default()};- let result = into_open_router(request, &model, None).unwrap();-- assert_eq!(- result.session_id,- open_router_session_id(Some(thread_id.into()))- );- assert_ne!(result.session_id.as_deref(), Some(thread_id));- assert_ne!(- result.session_id,- open_router_session_id(Some("another-thread-id".into()))- );+ for (requested, maximum, expected) in [+ (None, None, None),+ (None, Some(4096), Some(4096)),+ (Some(1024), Some(4096), Some(1024)),+ (Some(8192), Some(4096), Some(4096)),+ ] {+ let result = into_open_router(request(requested), &model, maximum).unwrap();+ assert_eq!(+ result.session_id,+ open_router_session_id(Some(thread_id.into()))+ );+ assert_ne!(result.session_id.as_deref(), Some(thread_id));+ assert_ne!(+ result.session_id,+ open_router_session_id(Some("another-thread-id".into()))+ );+ assert_eq!(+ serde_json::to_value(result).unwrap()["max_tokens"].as_u64(),+ expected+ );+ }}#[gpui::test]@@ -867,6 +879,7 @@ mod tests {prompt_id: None,intent: None,compact_at_tokens: None,+ max_output_tokens: None,};let result = into_open_router(request, &model, None).unwrap();@@ -1009,6 +1022,7 @@ mod tests {prompt_id: None,intent: None,compact_at_tokens: None,+ max_output_tokens: None,};let result = into_open_router(request, &model, None).unwrap();@@ -1075,6 +1089,7 @@ mod tests {prompt_id: None,intent: None,compact_at_tokens: None,+ max_output_tokens: None,};let result = into_open_router(request, &model, None).unwrap();diff --git a/crates/language_models/src/provider/opencode.rs b/crates/language_models/src/provider/opencode.rsindex 03b23520a0b..94b02441011 100644--- a/crates/language_models/src/provider/opencode.rs+++ b/crates/language_models/src/provider/opencode.rs@@ -750,6 +750,11 @@ impl LanguageModel for OpenCodeLanguageModel {.boxed()}ApiProtocol::Google => {+ let mut request = request;+ if request.max_output_tokens.is_some() {+ request.max_output_tokens =+ request.effective_max_output_tokens(self.max_output_tokens());+ }let mode = if self.supports_thinking() && request.thinking_allowed {google_ai::GoogleModelMode::Thinking {budget_tokens: None,diff --git a/crates/language_models_cloud/src/language_models_cloud.rs b/crates/language_models_cloud/src/language_models_cloud.rsindex 8808d6514f0..13b899317c8 100644--- a/crates/language_models_cloud/src/language_models_cloud.rs+++ b/crates/language_models_cloud/src/language_models_cloud.rs@@ -609,6 +609,11 @@ impl<TP: CloudLlmTokenProvider + 'static> LanguageModel for CloudLanguageModel<T) && self.model.supports_server_side_compaction}+ fn supports_explicit_compaction_output_limit(&self) -> bool {+ self.model.provider == cloud_llm_client::LanguageModelProvider::Anthropic+ && self.supports_explicit_compaction()+ }+fn minimum_explicit_compaction_input_tokens(&self) -> Option<u64> {(self.model.provider == cloud_llm_client::LanguageModelProvider::Anthropic&& self.supports_explicit_compaction())@@ -710,6 +715,11 @@ impl<TP: CloudLlmTokenProvider + 'static> LanguageModel for CloudLanguageModel<T.boxed();}+ let mut request = request;+ if request.max_output_tokens.is_some() {+ request.max_output_tokens =+ request.effective_max_output_tokens(self.max_output_tokens());+ }let thread_id = request.thread_id.clone();let prompt_id = request.prompt_id.clone();let app_version = self.app_version.clone();@@ -1328,6 +1338,8 @@ mod tests {}});let model = cloud_test_model(http_client);+ assert!(model.supports_explicit_compaction());+ assert!(!model.supports_explicit_compaction_output_limit());let request = compact_test_request();let result = model.compact(request, &cx.to_async()).await.unwrap();@@ -1475,10 +1487,9 @@ mod tests {});let model = cloud_anthropic_test_model(http_client);- let result = model- .compact(compact_test_request(), &cx.to_async())- .await- .unwrap();+ let mut request = compact_test_request();+ request.max_output_tokens = Some(8192);+ let result = model.compact(request, &cx.to_async()).await.unwrap();assert_eq!(result.usage,@@ -1510,6 +1521,7 @@ mod tests {assert_eq!(uri, "http://test.example/completions?");let body = serde_json::from_str::<serde_json::Value>(&body).unwrap();assert_eq!(body["provider"], "anthropic");+ assert_eq!(body["provider_request"]["max_tokens"], 8192);assert_eq!(body["provider_request"]["context_management"],json!({@@ -1805,12 +1817,63 @@ mod tests {let model = cloud_anthropic_test_model(FakeHttpClient::with_404_response());assert!(model.supports_explicit_compaction());+ assert!(model.supports_explicit_compaction_output_limit());+ assert_eq!(model.max_total_tokens(), Some(model.max_token_count()));assert_eq!(model.minimum_explicit_compaction_input_tokens(),Some(anthropic::MIN_COMPACTION_TRIGGER_TOKENS));}+ #[gpui::test]+ async fn hosted_open_ai_preserves_unset_output_and_clamps_explicit_caps(+ cx: &mut gpui::TestAppContext,+ ) {+ for (limit, expected) in [+ (None, None),+ (Some(8192), Some(8192)),+ (Some(u64::MAX), Some(128_000)),+ ] {+ let model = cloud_test_model(FakeHttpClient::create(move |mut request| async move {+ assert_eq!(request.uri().path(), "/completions");+ let mut body = String::new();+ request.body_mut().read_to_string(&mut body).await?;+ let body: serde_json::Value = serde_json::from_str(&body)?;+ assert_eq!(+ body["provider_request"].get("max_output_tokens").cloned(),+ expected.map(|value| json!(value))+ );+ let completed = CompletionEvent::Event(json!({+ "type": "response.completed",+ "response": {+ "id": "response-1", "status": "completed", "output": [],+ "usage": {"input_tokens": 1, "output_tokens": 0, "total_tokens": 1}+ }+ }));+ let ended = CompletionEvent::<serde_json::Value>::Status(+ CompletionRequestStatus::StreamEnded,+ );+ Ok(Response::builder()+ .status(200)+ .header(SERVER_SUPPORTS_STATUS_MESSAGES_HEADER_NAME, "true")+ .body(AsyncBody::from(format!(+ "{}\n{}\n",+ serde_json::to_string(&completed)?,+ serde_json::to_string(&ended)?,+ )))?)+ }));+ let mut request = compact_test_request();+ request.max_output_tokens = limit;+ let mut stream = model+ .stream_completion(request, &cx.to_async())+ .await+ .unwrap();+ while let Some(event) = stream.next().await {+ event.unwrap();+ }+ }+ }+fn compact_test_request() -> LanguageModelRequest {LanguageModelRequest {thread_id: Some("thread-123".to_string()),diff --git a/crates/open_ai/src/completion.rs b/crates/open_ai/src/completion.rsindex cf087070622..37f73bdfd9f 100644--- a/crates/open_ai/src/completion.rs+++ b/crates/open_ai/src/completion.rs@@ -58,6 +58,7 @@ pub fn into_open_ai(reasoning_effort: Option<ReasoningEffort>,interleaved_reasoning: bool,) -> Result<crate::Request> {+ let max_output_tokens = request.effective_max_output_tokens(max_output_tokens);if request.tools.iter()@@ -257,6 +258,7 @@ pub fn into_open_ai_response(supports_none_reasoning_effort: bool,compaction_state_owner: &LanguageModelProviderId,) -> Result<ResponseRequest> {+ let max_output_tokens = request.effective_max_output_tokens(max_output_tokens);let stream = !model_id.starts_with("o1-");let LanguageModelRequest {@@ -272,6 +274,7 @@ pub fn into_open_ai_response(thinking_effort,speed,compact_at_tokens,+ max_output_tokens: _,} = request;let service_tier = service_tier_for(speed);@@ -1834,6 +1837,7 @@ mod tests {thinking_effort: Some("high".into()),speed: None,compact_at_tokens: None,+ max_output_tokens: None,};let response = into_open_ai_response(@@ -2020,6 +2024,7 @@ mod tests {thinking_effort: None,speed: None,compact_at_tokens: None,+ max_output_tokens: None,};let response = into_open_ai_response(@@ -2121,6 +2126,7 @@ mod tests {thinking_effort: None,speed: None,compact_at_tokens: None,+ max_output_tokens: None,};let response = into_open_ai_response(@@ -2206,6 +2212,7 @@ mod tests {thinking_effort: None,speed: None,compact_at_tokens: None,+ max_output_tokens: None,};let response = into_open_ai_response(@@ -2272,6 +2279,7 @@ mod tests {thinking_effort: Some("high".into()),speed: None,compact_at_tokens: None,+ max_output_tokens: None,};let response = into_open_ai_response(@@ -2318,6 +2326,7 @@ mod tests {thinking_effort: None,speed,compact_at_tokens: None,+ max_output_tokens: None,};let response = into_open_ai_response(@@ -2370,6 +2379,7 @@ mod tests {thinking_effort: None,speed,compact_at_tokens: None,+ max_output_tokens: None,};let chat = into_open_ai(@@ -2415,22 +2425,60 @@ mod tests {thinking_effort: None,speed: None,compact_at_tokens: None,+ max_output_tokens: None,};- let chat = into_open_ai(- request,- "compatible-model",- false,- false,- Some(4096),- ChatCompletionMaxTokensParameter::MaxTokens,- None,- false,- )?;+ for (requested, model_maximum, expected) in [+ (None, None, None),+ (None, Some(4096), Some(4096)),+ (Some(1024), Some(4096), Some(1024)),+ (Some(8192), Some(4096), Some(4096)),+ (Some(1024), None, Some(1024)),+ ] {+ let mut request = request.clone();+ request.max_output_tokens = requested;+ for (parameter, field, absent_field) in [+ (+ ChatCompletionMaxTokensParameter::MaxCompletionTokens,+ "max_completion_tokens",+ "max_tokens",+ ),+ (+ ChatCompletionMaxTokensParameter::MaxTokens,+ "max_tokens",+ "max_completion_tokens",+ ),+ ] {+ let chat = into_open_ai(+ request.clone(),+ "compatible-model",+ false,+ false,+ model_maximum,+ parameter,+ None,+ false,+ )?;+ let serialized = serde_json::to_value(chat)?;+ assert_eq!(serialized[field].as_u64(), expected);+ assert!(serialized.get(absent_field).is_none());+ }- let serialized = serde_json::to_value(&chat)?;- assert_eq!(serialized.get("max_completion_tokens"), None);- assert_eq!(serialized["max_tokens"], json!(4096));+ let response = into_open_ai_response(+ request,+ "gpt-4.1",+ false,+ false,+ model_maximum,+ None,+ false,+ &language_model_core::OPEN_AI_PROVIDER_ID,+ )?;+ assert_eq!(+ serde_json::to_value(response)?["max_output_tokens"].as_u64(),+ expected+ );+ }Ok(())}@@ -2454,6 +2502,7 @@ mod tests {thinking_effort: Some("high".into()),speed: None,compact_at_tokens: None,+ max_output_tokens: None,};let response = into_open_ai_response(@@ -2495,6 +2544,7 @@ mod tests {thinking_effort: Some("none".into()),speed: None,compact_at_tokens: None,+ max_output_tokens: None,};let response = into_open_ai_response(@@ -2548,6 +2598,7 @@ mod tests {thinking_effort: None,speed: None,compact_at_tokens: None,+ max_output_tokens: None,};let response = into_open_ai_response(@@ -2640,6 +2691,7 @@ mod tests {thinking_effort: None,speed: None,compact_at_tokens: None,+ max_output_tokens: None,};let response = into_open_ai_response(@@ -2730,6 +2782,7 @@ mod tests {thinking_effort: None,speed: None,compact_at_tokens: None,+ max_output_tokens: None,};let response = into_open_ai_response(@@ -4034,6 +4087,7 @@ mod tests {thinking_effort: None,speed: None,compact_at_tokens: None,+ max_output_tokens: None,};let result = into_open_ai(