Skip to content

armgit / zed

Public Imported from GitHub
0

Code at the speed of thought – Zed is a high-performance, multiplayer code editor from the creators of Atom and Tree-sitter.

Add per-request model output limits (#64047)

MartinYe123459d996d8dbae
diff --git a/crates/agent/src/thread.rs b/crates/agent/src/thread.rs
index a1a5409ee1c..39f99aac0ca 100644
--- a/crates/agent/src/thread.rs
+++ b/crates/agent/src/thread.rs
@@ -4150,6 +4150,7 @@ impl Thread {
thinking_effort: self.thinking_effort.clone(),
speed: self.speed(),
compact_at_tokens: None,
+ max_output_tokens: None,
};
log::debug!("Completion request built successfully");
diff --git a/crates/agent_ui/src/buffer_codegen.rs b/crates/agent_ui/src/buffer_codegen.rs
index bff276966c0..e0d3b95277e 100644
--- a/crates/agent_ui/src/buffer_codegen.rs
+++ b/crates/agent_ui/src/buffer_codegen.rs
@@ -557,6 +557,7 @@ impl CodegenAlternative {
thinking_effort: None,
speed: None,
compact_at_tokens: None,
+ max_output_tokens: None,
}
}))
}
@@ -639,6 +640,7 @@ impl CodegenAlternative {
thinking_effort: None,
speed: None,
compact_at_tokens: None,
+ max_output_tokens: None,
}
}))
}
diff --git a/crates/agent_ui/src/terminal_inline_assistant.rs b/crates/agent_ui/src/terminal_inline_assistant.rs
index 800a68a3f3c..efbd1d24949 100644
--- a/crates/agent_ui/src/terminal_inline_assistant.rs
+++ b/crates/agent_ui/src/terminal_inline_assistant.rs
@@ -276,6 +276,7 @@ impl TerminalInlineAssistant {
thinking_effort: None,
speed: None,
compact_at_tokens: None,
+ max_output_tokens: None,
}
}))
}
diff --git a/crates/anthropic/src/completion.rs b/crates/anthropic/src/completion.rs
index d2a601ad55c..ed909521ae5 100644
--- a/crates/anthropic/src/completion.rs
+++ b/crates/anthropic/src/completion.rs
@@ -290,6 +290,9 @@ pub fn into_anthropic(
cache_mode: AnthropicPromptCacheMode,
compaction_state_owner: &LanguageModelProviderId,
) -> Result<crate::Request> {
+ let max_output_tokens = request
+ .max_output_tokens
+ .map_or(max_output_tokens, |limit| limit.min(max_output_tokens));
let mut new_messages: Vec<Message> = Vec::new();
let mut system_message = String::new();
let mut any_message_wants_cache = false;
@@ -907,6 +910,7 @@ mod tests {
thinking_effort: None,
speed: None,
compact_at_tokens: None,
+ max_output_tokens: Some(1024),
};
let anthropic_request = into_anthropic(
@@ -920,6 +924,10 @@ mod tests {
)
.unwrap();
+ assert_eq!(
+ serde_json::to_value(&anthropic_request).unwrap()["max_tokens"],
+ 1024
+ );
// No message content block should carry cache_control anymore; the
// conversation breakpoint is set via top-level automatic caching.
assert_eq!(anthropic_request.messages.len(), 1);
@@ -1015,6 +1023,7 @@ mod tests {
thinking_effort: None,
speed: None,
compact_at_tokens: None,
+ max_output_tokens: Some(8192),
};
let anthropic_request = into_anthropic(
@@ -1028,6 +1037,10 @@ mod tests {
)
.unwrap();
+ assert_eq!(
+ serde_json::to_value(&anthropic_request).unwrap()["max_tokens"],
+ 4096
+ );
assert!(anthropic_request.cache_control.is_none());
assert!(matches!(
anthropic_request.system,
@@ -1075,6 +1088,7 @@ mod tests {
thinking_effort: Some("xhigh".into()),
speed: None,
compact_at_tokens: None,
+ max_output_tokens: None,
};
let anthropic_request = into_anthropic(
@@ -1126,6 +1140,7 @@ mod tests {
thinking_effort: None,
speed: None,
compact_at_tokens: None,
+ max_output_tokens: None,
};
let anthropic_request = into_anthropic(
@@ -1195,6 +1210,7 @@ mod tests {
thinking_effort: None,
speed: None,
compact_at_tokens: None,
+ max_output_tokens: None,
};
let anthropic_request = into_anthropic(
@@ -1261,6 +1277,7 @@ mod tests {
thinking_effort: None,
speed: None,
compact_at_tokens: None,
+ max_output_tokens: None,
};
let anthropic_request = into_anthropic(
@@ -1274,6 +1291,10 @@ mod tests {
)
.unwrap();
+ assert_eq!(
+ serde_json::to_value(&anthropic_request).unwrap()["max_tokens"],
+ 4096
+ );
assert!(anthropic_request.cache_control.is_none());
assert!(matches!(
anthropic_request.system,
@@ -1301,6 +1322,7 @@ mod tests {
thinking_allowed: true,
speed: None,
compact_at_tokens: None,
+ max_output_tokens: None,
};
request.messages.push(LanguageModelRequestMessage {
role: Role::Assistant,
diff --git a/crates/copilot_chat/src/copilot_chat.rs b/crates/copilot_chat/src/copilot_chat.rs
index 92073f5c0ba..71ef0895d96 100644
--- a/crates/copilot_chat/src/copilot_chat.rs
+++ b/crates/copilot_chat/src/copilot_chat.rs
@@ -279,8 +279,9 @@ impl Model {
self.capabilities.limits.max_context_window_tokens as u64
}
- pub fn max_output_tokens(&self) -> usize {
- self.capabilities.limits.max_output_tokens
+ pub fn max_output_tokens(&self) -> Option<u64> {
+ let limit = self.capabilities.limits.max_output_tokens as u64;
+ (limit > 0).then_some(limit)
}
pub fn supports_tools(&self) -> bool {
@@ -356,6 +357,8 @@ pub struct Request {
pub temperature: f32,
pub model: String,
pub messages: Vec<ChatMessage>,
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ pub max_tokens: Option<u64>,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub tools: Vec<Tool>,
#[serde(default, skip_serializing_if = "Option::is_none")]
diff --git a/crates/copilot_chat/src/model.rs b/crates/copilot_chat/src/model.rs
index bbd12c4d5aa..12d524c318a 100644
--- a/crates/copilot_chat/src/model.rs
+++ b/crates/copilot_chat/src/model.rs
@@ -161,6 +161,12 @@ impl LanguageModel for CopilotChatLanguageModel {
let copilot_chat = self.copilot_chat.clone();
let request_limiter = self.request_limiter.clone();
let future = cx.spawn(async move |cx| {
+ let max_output_tokens = model
+ .max_output_tokens()
+ .or(request.max_output_tokens)
+ .ok_or_else(|| {
+ anyhow!("Copilot did not provide an output limit for this model")
+ })?;
let effort = request
.thinking_effort
.as_ref()
@@ -170,7 +176,7 @@ impl LanguageModel for CopilotChatLanguageModel {
request,
model.id().to_string(),
0.0,
- model.max_output_tokens() as u64,
+ max_output_tokens,
if model.supports_adaptive_thinking() {
AnthropicModelMode::Thinking {
budget_tokens: None,
@@ -180,7 +186,7 @@ impl LanguageModel for CopilotChatLanguageModel {
budget_tokens: compute_thinking_budget(
model.min_thinking_budget(),
model.max_thinking_budget(),
- model.max_output_tokens() as u32,
+ max_output_tokens.min(u32::MAX as u64) as u32,
),
}
} else {
@@ -814,6 +820,9 @@ fn into_copilot_chat(
model: &CopilotChatModel,
request: LanguageModelRequest,
) -> Result<CopilotChatRequest> {
+ let max_tokens = request
+ .max_output_tokens
+ .and_then(|_| request.effective_max_output_tokens(model.max_output_tokens()));
let temperature = request.temperature;
let tool_choice = request.tool_choice;
let thinking_allowed = request.thinking_allowed;
@@ -1008,6 +1017,7 @@ fn into_copilot_chat(
temperature: temperature.unwrap_or(0.1),
model: model.id().to_string(),
messages,
+ max_tokens,
tools,
tool_choice: tool_choice.map(|choice| match choice {
LanguageModelToolChoice::Auto => ToolChoice::Auto,
@@ -1015,11 +1025,13 @@ fn into_copilot_chat(
LanguageModelToolChoice::None => ToolChoice::None,
}),
thinking_budget: if thinking_allowed && model.supports_thinking() {
- compute_thinking_budget(
- model.min_thinking_budget(),
- model.max_thinking_budget(),
- model.max_output_tokens() as u32,
- )
+ model.max_output_tokens().or(max_tokens).and_then(|limit| {
+ compute_thinking_budget(
+ model.min_thinking_budget(),
+ model.max_thinking_budget(),
+ limit.min(u32::MAX as u64) as u32,
+ )
+ })
} else {
None
},
@@ -1064,6 +1076,9 @@ fn into_copilot_responses(
) -> Result<copilot_responses::Request> {
use copilot_responses as responses;
+ let max_output_tokens = request
+ .max_output_tokens
+ .and_then(|_| request.effective_max_output_tokens(model.max_output_tokens()));
let LanguageModelRequest {
thread_id: _,
prompt_id: _,
@@ -1077,6 +1092,7 @@ fn into_copilot_responses(
thinking_effort,
speed: _,
compact_at_tokens: _,
+ max_output_tokens: _,
} = request;
let mut input_items: Vec<responses::ResponseInputItem> = Vec::new();
@@ -1255,6 +1271,7 @@ fn into_copilot_responses(
input: input_items,
stream: model.uses_streaming(),
temperature,
+ max_output_tokens,
tools: converted_tools,
tool_choice: mapped_tool_choice,
reasoning: if thinking_allowed {
@@ -1523,6 +1540,54 @@ mod tests {
);
}
+ #[test]
+ fn request_output_limits_reach_copilot_payloads() -> Result<()> {
+ let model = test_responses_model();
+ for (limit, expected) in [
+ (None, None),
+ (Some(1024), Some(1024)),
+ (Some(u64::MAX), Some(4096)),
+ ] {
+ let request = LanguageModelRequest {
+ max_output_tokens: limit,
+ ..Default::default()
+ };
+ let responses = serde_json::to_value(into_copilot_responses(&model, request.clone())?)?;
+ let chat = serde_json::to_value(into_copilot_chat(&model, request)?)?;
+ assert_eq!(
+ responses.get("max_output_tokens").cloned(),
+ expected.map(|value| json!(value))
+ );
+ assert_eq!(
+ chat.get("max_tokens").cloned(),
+ expected.map(|value| json!(value))
+ );
+ }
+ let mut value = serde_json::to_value(test_responses_model())?;
+ value["capabilities"]["limits"]
+ .as_object_mut()
+ .unwrap()
+ .remove("max_output_tokens");
+ let model = serde_json::from_value(value)?;
+ for limit in [None, Some(1024)] {
+ let request = LanguageModelRequest {
+ max_output_tokens: limit,
+ ..Default::default()
+ };
+ let responses = serde_json::to_value(into_copilot_responses(&model, request.clone())?)?;
+ let chat = serde_json::to_value(into_copilot_chat(&model, request)?)?;
+ assert_eq!(
+ responses.get("max_output_tokens").cloned(),
+ limit.map(|value| json!(value))
+ );
+ assert_eq!(
+ chat.get("max_tokens").cloned(),
+ limit.map(|value| json!(value))
+ );
+ }
+ Ok(())
+ }
+
#[test]
fn into_copilot_responses_replays_reasoning_details() {
let model = test_responses_model();
diff --git a/crates/copilot_chat/src/responses.rs b/crates/copilot_chat/src/responses.rs
index 5cb8cf0af4a..597b058146e 100644
--- a/crates/copilot_chat/src/responses.rs
+++ b/crates/copilot_chat/src/responses.rs
@@ -16,6 +16,8 @@ pub struct Request {
pub stream: bool,
#[serde(skip_serializing_if = "Option::is_none")]
pub temperature: Option<f32>,
+ #[serde(skip_serializing_if = "Option::is_none")]
+ pub max_output_tokens: Option<u64>,
#[serde(skip_serializing_if = "Vec::is_empty")]
pub tools: Vec<ToolDefinition>,
#[serde(skip_serializing_if = "Option::is_none")]
diff --git a/crates/git_ui/src/git_panel.rs b/crates/git_ui/src/git_panel.rs
index 7f10a4a8829..26126aeaa91 100644
--- a/crates/git_ui/src/git_panel.rs
+++ b/crates/git_ui/src/git_panel.rs
@@ -4166,6 +4166,7 @@ impl GitPanel {
thinking_effort: None,
speed: None,
compact_at_tokens: None,
+ max_output_tokens: None,
};
let stream = model.stream_completion_text(request, cx);
diff --git a/crates/google_ai/src/completion.rs b/crates/google_ai/src/completion.rs
index 60ee81751b5..fda7eb16354 100644
--- a/crates/google_ai/src/completion.rs
+++ b/crates/google_ai/src/completion.rs
@@ -113,6 +113,10 @@ pub fn into_google(
}
let thinking_config = thinking_config_for_request(&request, &model_id, mode);
+ let max_output_tokens = request
+ .effective_max_output_tokens(None)
+ .map(usize::try_from)
+ .transpose()?;
let system_instructions = if request
.messages
@@ -179,7 +183,7 @@ pub fn into_google(
generation_config: Some(GenerationConfig {
candidate_count: Some(1),
stop_sequences: Some(request.stop),
- max_output_tokens: None,
+ max_output_tokens,
temperature: request.temperature.map(|t| t as f64),
thinking_config,
top_p: None,
@@ -516,6 +520,11 @@ mod tests {
)
.unwrap();
+ assert!(
+ serde_json::to_value(&request).unwrap()["generationConfig"]
+ .get("maxOutputTokens")
+ .is_none()
+ );
let thinking_config = request.generation_config.unwrap().thinking_config.unwrap();
assert_eq!(thinking_config.include_thoughts, Some(true));
assert_eq!(thinking_config.thinking_level, Some(ThinkingLevel::Low));
@@ -542,6 +551,7 @@ mod tests {
}
});
let mut request = text_request();
+ request.max_output_tokens = Some(1024);
request.tools = vec![LanguageModelRequestTool::function(
"grep".to_string(),
"Search files".to_string(),
@@ -556,6 +566,7 @@ mod tests {
)
.unwrap();
let serialized = serde_json::to_value(request).unwrap();
+ assert_eq!(serialized["generationConfig"]["maxOutputTokens"], 1024);
let declaration = &serialized["tools"][0]["functionDeclarations"][0];
assert_eq!(declaration["parametersJsonSchema"], input_schema);
@@ -566,6 +577,7 @@ mod tests {
fn into_google_turns_off_budget_thinking_when_supported() {
let mut request = text_request();
request.thinking_allowed = false;
+ request.max_output_tokens = Some(0);
let request = into_google(
request,
@@ -576,6 +588,10 @@ mod tests {
)
.unwrap();
+ assert_eq!(
+ serde_json::to_value(&request).unwrap()["generationConfig"]["maxOutputTokens"],
+ 0
+ );
let thinking_config = request.generation_config.unwrap().thinking_config.unwrap();
assert_eq!(thinking_config.thinking_budget, Some(0));
assert_eq!(thinking_config.include_thoughts, None);
diff --git a/crates/language_model/src/language_model.rs b/crates/language_model/src/language_model.rs
index d4bb4d5f13d..38c89fd3e15 100644
--- a/crates/language_model/src/language_model.rs
+++ b/crates/language_model/src/language_model.rs
@@ -171,6 +171,11 @@ pub trait LanguageModel: Send + Sync {
false
}
+ /// Whether native compaction honors `LanguageModelRequest::max_output_tokens`.
+ fn supports_explicit_compaction_output_limit(&self) -> bool {
+ false
+ }
+
/// The provider-enforced input size required for explicit compaction.
fn minimum_explicit_compaction_input_tokens(&self) -> Option<u64> {
None
@@ -210,7 +215,18 @@ pub trait LanguageModel: Send + Sync {
false
}
+ /// Returns the input token ceiling.
fn max_token_count(&self) -> u64;
+
+ /// Returns the combined input and output ceiling, if one applies.
+ ///
+ /// The conservative default shares the input ceiling with output. `None`
+ /// means generation does not consume that window, not merely that the API
+ /// validates input separately or stops generation at the window boundary.
+ fn max_total_tokens(&self) -> Option<u64> {
+ Some(self.max_token_count())
+ }
+
fn max_output_tokens(&self) -> Option<u64> {
None
}
diff --git a/crates/language_model_core/src/request.rs b/crates/language_model_core/src/request.rs
index 68222eff967..a6426235dfc 100644
--- a/crates/language_model_core/src/request.rs
+++ b/crates/language_model_core/src/request.rs
@@ -473,9 +473,23 @@ pub struct LanguageModelRequest {
pub speed: Option<Speed>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub compact_at_tokens: Option<u64>,
+ /// An output-token ceiling, including reasoning tokens where the provider counts them.
+ ///
+ /// Providers must forward this limit (clamped to a known model maximum), or reject
+ /// capped requests when unsupported. `None` preserves the provider's default.
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ pub max_output_tokens: Option<u64>,
}
impl LanguageModelRequest {
+ /// Combines the request's output limit with the model's output limit.
+ pub fn effective_max_output_tokens(&self, model_maximum: Option<u64>) -> Option<u64> {
+ match (self.max_output_tokens, model_maximum) {
+ (Some(request), Some(model)) => Some(request.min(model)),
+ (request, model) => request.or(model),
+ }
+ }
+
pub fn contains_custom_tool_input(&self) -> bool {
self.tools
.iter()
@@ -523,6 +537,29 @@ pub struct LanguageModelResponseMessage {
mod tests {
use super::*;
+ #[test]
+ fn request_output_limit_serialization() -> serde_json::Result<()> {
+ let request = LanguageModelRequest::default();
+ let mut serialized = serde_json::to_value(&request)?;
+ assert!(serialized.get("max_output_tokens").is_none());
+ assert_eq!(
+ serde_json::from_value::<LanguageModelRequest>(serialized.clone())?,
+ request
+ );
+
+ serialized["max_output_tokens"] = serde_json::json!(1024);
+ let capped: LanguageModelRequest = serde_json::from_value(serialized.clone())?;
+ assert_eq!(capped.max_output_tokens, Some(1024));
+ assert_eq!(serde_json::to_value(capped)?, serialized);
+
+ serialized["max_output_tokens"] = serde_json::Value::Null;
+ assert_eq!(
+ serde_json::from_value::<LanguageModelRequest>(serialized)?,
+ request
+ );
+ Ok(())
+ }
+
#[test]
fn test_language_model_tool_result_content_deserialization() {
// Test plain string
diff --git a/crates/language_models/src/provider/anthropic.rs b/crates/language_models/src/provider/anthropic.rs
index 73b374302f6..686ff2b702f 100644
--- a/crates/language_models/src/provider/anthropic.rs
+++ b/crates/language_models/src/provider/anthropic.rs
@@ -467,6 +467,7 @@ mod tests {
let model = direct_anthropic_test_model(&provider);
assert!(model.supports_explicit_compaction());
+ assert_eq!(model.max_total_tokens(), Some(model.max_token_count()));
assert_eq!(
model.minimum_explicit_compaction_input_tokens(),
Some(anthropic::MIN_COMPACTION_TRIGGER_TOKENS)
@@ -570,6 +571,7 @@ mod tests {
store_key.await.unwrap();
let model = direct_anthropic_test_model(&provider);
let request = LanguageModelRequest {
+ max_output_tokens: Some(8192),
messages: vec![LanguageModelRequestMessage {
role: language_model::Role::User,
content: vec![MessageContent::Text("Retain this context.".to_string())],
@@ -614,6 +616,7 @@ mod tests {
.is_some_and(|header| header.contains(anthropic::COMPACTION_BETA_HEADER))
);
let body = serde_json::from_str::<serde_json::Value>(&body).unwrap();
+ assert_eq!(body["max_tokens"], 8192);
assert_eq!(
body["context_management"],
json!({
@@ -798,6 +801,10 @@ impl LanguageModel for AnthropicModel {
self.model.supports_compaction
}
+ fn supports_explicit_compaction_output_limit(&self) -> bool {
+ self.supports_explicit_compaction()
+ }
+
fn minimum_explicit_compaction_input_tokens(&self) -> Option<u64> {
self.supports_explicit_compaction()
.then_some(anthropic::MIN_COMPACTION_TRIGGER_TOKENS)
diff --git a/crates/language_models/src/provider/bedrock.rs b/crates/language_models/src/provider/bedrock.rs
index d3b5c41efbe..092dcd1ae10 100644
--- a/crates/language_models/src/provider/bedrock.rs
+++ b/crates/language_models/src/provider/bedrock.rs
@@ -2004,6 +2004,9 @@ pub fn into_bedrock(
guardrail_identifier: Option<String>,
guardrail_version: Option<String>,
) -> Result<bedrock::Request> {
+ let max_output_tokens = request
+ .max_output_tokens
+ .map_or(max_output_tokens, |limit| limit.min(max_output_tokens));
if request.contains_custom_tool_input() {
anyhow::bail!("Bedrock does not support custom tools");
}
@@ -2948,10 +2951,15 @@ mod tests {
// Claude Opus 5 runs adaptive thinking by default when the `thinking`
// field is omitted, so suppressing thinking requires an explicit
// `disabled` opt-out. Earlier Claude models treat omission as "off".
- for (model, expects_explicit_opt_out) in [
- ("us.anthropic.claude-opus-5", true),
- ("global.anthropic.claude-opus-5", true),
- ("us.anthropic.claude-opus-4-8", false),
+ for (model, expects_explicit_opt_out, output_limit, expected_output) in [
+ ("us.anthropic.claude-opus-5", true, None, 128_000),
+ ("global.anthropic.claude-opus-5", true, Some(8192), 8192),
+ (
+ "us.anthropic.claude-opus-4-8",
+ false,
+ Some(256_000),
+ 128_000,
+ ),
] {
let request = into_bedrock(
LanguageModelRequest {
@@ -2962,6 +2970,7 @@ mod tests {
reasoning_details: None,
}],
thinking_allowed: false,
+ max_output_tokens: output_limit,
..Default::default()
},
model.to_string(),
@@ -2977,6 +2986,7 @@ mod tests {
)
.unwrap();
+ assert_eq!(request.max_tokens, expected_output);
if expects_explicit_opt_out {
assert!(
matches!(request.thinking, Some(bedrock::Thinking::Disabled)),
diff --git a/crates/language_models/src/provider/deepseek.rs b/crates/language_models/src/provider/deepseek.rs
index b2365fbbc61..19a988adc2e 100644
--- a/crates/language_models/src/provider/deepseek.rs
+++ b/crates/language_models/src/provider/deepseek.rs
@@ -366,6 +366,7 @@ pub fn into_deepseek(
model: &deepseek::Model,
max_output_tokens: Option<u64>,
) -> Result<deepseek::Request> {
+ let max_output_tokens = request.effective_max_output_tokens(max_output_tokens);
if request.contains_custom_tool_input() {
anyhow::bail!("DeepSeek does not support custom tools");
}
@@ -666,3 +667,24 @@ impl DeepSeekEventMapper {
events
}
}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn request_output_limits_reach_deepseek_payloads() -> Result<()> {
+ for (limit, expected) in [(None, 4096), (Some(1024), 1024), (Some(8192), 4096)] {
+ let request = into_deepseek(
+ LanguageModelRequest {
+ max_output_tokens: limit,
+ ..Default::default()
+ },
+ &deepseek::Model::V4Pro,
+ Some(4096),
+ )?;
+ assert_eq!(serde_json::to_value(request)?["max_tokens"], expected);
+ }
+ Ok(())
+ }
+}
diff --git a/crates/language_models/src/provider/google.rs b/crates/language_models/src/provider/google.rs
index f7a50f8ee94..57e00f8d021 100644
--- a/crates/language_models/src/provider/google.rs
+++ b/crates/language_models/src/provider/google.rs
@@ -354,6 +354,11 @@ impl LanguageModel for GoogleLanguageModel {
LanguageModelCompletionError,
>,
> {
+ let mut request = request;
+ if request.max_output_tokens.is_some() {
+ request.max_output_tokens =
+ request.effective_max_output_tokens(self.max_output_tokens());
+ }
let request = match into_google(
request,
self.model.request_id().to_string(),
diff --git a/crates/language_models/src/provider/llama_cpp.rs b/crates/language_models/src/provider/llama_cpp.rs
index 3b18b732253..c21494ffc4f 100644
--- a/crates/language_models/src/provider/llama_cpp.rs
+++ b/crates/language_models/src/provider/llama_cpp.rs
@@ -853,8 +853,8 @@ fn build_llama_cpp_request(
model: model_name.to_string(),
messages,
stream: true,
- // Let the server decide the output length (its `n_predict` default).
- max_tokens: None,
+ // Without an explicit limit, preserve the server's `n_predict` default.
+ max_tokens: request.max_output_tokens.map(i32::try_from).transpose()?,
stop: if request.stop.is_empty() {
None
} else {
@@ -1702,6 +1702,7 @@ mod tests {
supports_thinking: true,
},
LanguageModelRequest {
+ max_output_tokens: Some(1024),
messages: vec![language_model::LanguageModelRequestMessage {
role: Role::Assistant,
content: vec![
@@ -1719,6 +1720,7 @@ mod tests {
)
.unwrap();
+ assert_eq!(serde_json::to_value(&request).unwrap()["max_tokens"], 1024);
assert_eq!(request.messages.len(), 1);
match &request.messages[0] {
llama_cpp::ChatMessage::Assistant {
@@ -1763,6 +1765,12 @@ mod tests {
)
.unwrap();
+ assert!(
+ serde_json::to_value(&request)
+ .unwrap()
+ .get("max_tokens")
+ .is_none()
+ );
assert_eq!(request.messages.len(), 1);
match &request.messages[0] {
llama_cpp::ChatMessage::Assistant {
diff --git a/crates/language_models/src/provider/lmstudio.rs b/crates/language_models/src/provider/lmstudio.rs
index a2de082c1b4..310932cfc45 100644
--- a/crates/language_models/src/provider/lmstudio.rs
+++ b/crates/language_models/src/provider/lmstudio.rs
@@ -425,7 +425,13 @@ impl LmStudioLanguageModel {
stream_options: Some(lmstudio::StreamOptions {
include_usage: true,
}),
- max_tokens: Some(-1),
+ max_tokens: Some(
+ request
+ .max_output_tokens
+ .map(i32::try_from)
+ .transpose()?
+ .unwrap_or(-1),
+ ),
stop: Some(request.stop),
// In LM Studio you can configure specific settings you'd like to use for your model.
// For example Qwen3 is recommended to be used with 0.7 temperature.
diff --git a/crates/language_models/src/provider/mistral.rs b/crates/language_models/src/provider/mistral.rs
index d447cb12eee..0ebb4a1b698 100644
--- a/crates/language_models/src/provider/mistral.rs
+++ b/crates/language_models/src/provider/mistral.rs
@@ -367,6 +367,7 @@ pub fn into_mistral(
model: mistral::Model,
max_output_tokens: Option<u64>,
) -> Result<(mistral::Request, Option<String>)> {
+ let max_output_tokens = request.effective_max_output_tokens(max_output_tokens);
if request.contains_custom_tool_input() {
anyhow::bail!("Mistral does not support custom tools");
}
@@ -850,7 +851,7 @@ mod tests {
#[test]
fn test_into_mistral_basic_conversion() {
- let request = LanguageModelRequest {
+ let request = |max_output_tokens| LanguageModelRequest {
messages: vec![
LanguageModelRequestMessage {
role: Role::System,
@@ -883,16 +884,32 @@ mod tests {
thinking_effort: None,
speed: Default::default(),
compact_at_tokens: None,
+ max_output_tokens,
};
- let (mistral_request, affinity) =
- into_mistral(request, mistral::Model::MistralSmallLatest, None).unwrap();
-
- assert_eq!(mistral_request.model, "mistral-small-latest");
- assert_eq!(mistral_request.temperature, Some(0.5));
- assert_eq!(mistral_request.messages.len(), 2);
- assert!(mistral_request.stream);
- assert_eq!(affinity, Some("abcdef".into()));
+ for (requested, maximum, expected) in [
+ (None, None, None),
+ (None, Some(4096), Some(4096)),
+ (Some(1024), Some(4096), Some(1024)),
+ (Some(8192), Some(4096), Some(4096)),
+ (Some(1024), None, Some(1024)),
+ ] {
+ let (mistral_request, affinity) = into_mistral(
+ request(requested),
+ mistral::Model::MistralSmallLatest,
+ maximum,
+ )
+ .unwrap();
+ assert_eq!(mistral_request.model, "mistral-small-latest");
+ assert_eq!(mistral_request.temperature, Some(0.5));
+ assert_eq!(mistral_request.messages.len(), 2);
+ assert!(mistral_request.stream);
+ assert_eq!(affinity, Some("abcdef".into()));
+ assert_eq!(
+ serde_json::to_value(mistral_request).unwrap()["max_tokens"].as_u64(),
+ expected
+ );
+ }
}
#[test]
@@ -915,6 +932,7 @@ mod tests {
thinking_effort: None,
speed: Default::default(),
compact_at_tokens: None,
+ max_output_tokens: None,
};
let (mistral_request, _) =
@@ -958,6 +976,7 @@ mod tests {
thinking_effort: None,
speed: None,
compact_at_tokens: None,
+ max_output_tokens: None,
};
let (mistral_request, _) =
diff --git a/crates/language_models/src/provider/ollama.rs b/crates/language_models/src/provider/ollama.rs
index 9ef157f9655..6dc0175c1b1 100644
--- a/crates/language_models/src/provider/ollama.rs
+++ b/crates/language_models/src/provider/ollama.rs
@@ -465,6 +465,7 @@ impl OllamaLanguageModel {
stream: true,
options: Some(ChatOptions {
num_ctx: Some(self.model.max_tokens),
+ num_predict: request.max_output_tokens.map(isize::try_from).transpose()?,
// Only send stop tokens if explicitly provided. When empty/None,
// Ollama will use the model's default stop tokens from its Modelfile.
// Sending an empty array would override and disable the defaults.
diff --git a/crates/language_models/src/provider/open_router.rs b/crates/language_models/src/provider/open_router.rs
index 84b4f7b898d..368ff34c3d9 100644
--- a/crates/language_models/src/provider/open_router.rs
+++ b/crates/language_models/src/provider/open_router.rs
@@ -458,6 +458,7 @@ pub fn into_open_router(
model: &Model,
max_output_tokens: Option<u64>,
) -> Result<open_router::Request> {
+ let max_output_tokens = request.effective_max_output_tokens(max_output_tokens);
if request.contains_custom_tool_input() {
anyhow::bail!("OpenRouter does not support custom tools");
}
@@ -790,8 +791,9 @@ mod tests {
None,
);
let thread_id = "internal-thread-id";
- let request = LanguageModelRequest {
+ let request = |max_output_tokens| LanguageModelRequest {
thread_id: Some(thread_id.to_string()),
+ max_output_tokens,
messages: vec![language_model::LanguageModelRequestMessage {
role: Role::User,
content: vec![MessageContent::Text("Hello".to_string())],
@@ -801,17 +803,27 @@ mod tests {
..Default::default()
};
- let result = into_open_router(request, &model, None).unwrap();
-
- assert_eq!(
- result.session_id,
- open_router_session_id(Some(thread_id.into()))
- );
- assert_ne!(result.session_id.as_deref(), Some(thread_id));
- assert_ne!(
- result.session_id,
- open_router_session_id(Some("another-thread-id".into()))
- );
+ for (requested, maximum, expected) in [
+ (None, None, None),
+ (None, Some(4096), Some(4096)),
+ (Some(1024), Some(4096), Some(1024)),
+ (Some(8192), Some(4096), Some(4096)),
+ ] {
+ let result = into_open_router(request(requested), &model, maximum).unwrap();
+ assert_eq!(
+ result.session_id,
+ open_router_session_id(Some(thread_id.into()))
+ );
+ assert_ne!(result.session_id.as_deref(), Some(thread_id));
+ assert_ne!(
+ result.session_id,
+ open_router_session_id(Some("another-thread-id".into()))
+ );
+ assert_eq!(
+ serde_json::to_value(result).unwrap()["max_tokens"].as_u64(),
+ expected
+ );
+ }
}
#[gpui::test]
@@ -867,6 +879,7 @@ mod tests {
prompt_id: None,
intent: None,
compact_at_tokens: None,
+ max_output_tokens: None,
};
let result = into_open_router(request, &model, None).unwrap();
@@ -1009,6 +1022,7 @@ mod tests {
prompt_id: None,
intent: None,
compact_at_tokens: None,
+ max_output_tokens: None,
};
let result = into_open_router(request, &model, None).unwrap();
@@ -1075,6 +1089,7 @@ mod tests {
prompt_id: None,
intent: None,
compact_at_tokens: None,
+ max_output_tokens: None,
};
let result = into_open_router(request, &model, None).unwrap();
diff --git a/crates/language_models/src/provider/opencode.rs b/crates/language_models/src/provider/opencode.rs
index 03b23520a0b..94b02441011 100644
--- a/crates/language_models/src/provider/opencode.rs
+++ b/crates/language_models/src/provider/opencode.rs
@@ -750,6 +750,11 @@ impl LanguageModel for OpenCodeLanguageModel {
.boxed()
}
ApiProtocol::Google => {
+ let mut request = request;
+ if request.max_output_tokens.is_some() {
+ request.max_output_tokens =
+ request.effective_max_output_tokens(self.max_output_tokens());
+ }
let mode = if self.supports_thinking() && request.thinking_allowed {
google_ai::GoogleModelMode::Thinking {
budget_tokens: None,
diff --git a/crates/language_models_cloud/src/language_models_cloud.rs b/crates/language_models_cloud/src/language_models_cloud.rs
index 8808d6514f0..13b899317c8 100644
--- a/crates/language_models_cloud/src/language_models_cloud.rs
+++ b/crates/language_models_cloud/src/language_models_cloud.rs
@@ -609,6 +609,11 @@ impl<TP: CloudLlmTokenProvider + 'static> LanguageModel for CloudLanguageModel<T
) && self.model.supports_server_side_compaction
}
+ fn supports_explicit_compaction_output_limit(&self) -> bool {
+ self.model.provider == cloud_llm_client::LanguageModelProvider::Anthropic
+ && self.supports_explicit_compaction()
+ }
+
fn minimum_explicit_compaction_input_tokens(&self) -> Option<u64> {
(self.model.provider == cloud_llm_client::LanguageModelProvider::Anthropic
&& self.supports_explicit_compaction())
@@ -710,6 +715,11 @@ impl<TP: CloudLlmTokenProvider + 'static> LanguageModel for CloudLanguageModel<T
.boxed();
}
+ let mut request = request;
+ if request.max_output_tokens.is_some() {
+ request.max_output_tokens =
+ request.effective_max_output_tokens(self.max_output_tokens());
+ }
let thread_id = request.thread_id.clone();
let prompt_id = request.prompt_id.clone();
let app_version = self.app_version.clone();
@@ -1328,6 +1338,8 @@ mod tests {
}
});
let model = cloud_test_model(http_client);
+ assert!(model.supports_explicit_compaction());
+ assert!(!model.supports_explicit_compaction_output_limit());
let request = compact_test_request();
let result = model.compact(request, &cx.to_async()).await.unwrap();
@@ -1475,10 +1487,9 @@ mod tests {
});
let model = cloud_anthropic_test_model(http_client);
- let result = model
- .compact(compact_test_request(), &cx.to_async())
- .await
- .unwrap();
+ let mut request = compact_test_request();
+ request.max_output_tokens = Some(8192);
+ let result = model.compact(request, &cx.to_async()).await.unwrap();
assert_eq!(
result.usage,
@@ -1510,6 +1521,7 @@ mod tests {
assert_eq!(uri, "http://test.example/completions?");
let body = serde_json::from_str::<serde_json::Value>(&body).unwrap();
assert_eq!(body["provider"], "anthropic");
+ assert_eq!(body["provider_request"]["max_tokens"], 8192);
assert_eq!(
body["provider_request"]["context_management"],
json!({
@@ -1805,12 +1817,63 @@ mod tests {
let model = cloud_anthropic_test_model(FakeHttpClient::with_404_response());
assert!(model.supports_explicit_compaction());
+ assert!(model.supports_explicit_compaction_output_limit());
+ assert_eq!(model.max_total_tokens(), Some(model.max_token_count()));
assert_eq!(
model.minimum_explicit_compaction_input_tokens(),
Some(anthropic::MIN_COMPACTION_TRIGGER_TOKENS)
);
}
+ #[gpui::test]
+ async fn hosted_open_ai_preserves_unset_output_and_clamps_explicit_caps(
+ cx: &mut gpui::TestAppContext,
+ ) {
+ for (limit, expected) in [
+ (None, None),
+ (Some(8192), Some(8192)),
+ (Some(u64::MAX), Some(128_000)),
+ ] {
+ let model = cloud_test_model(FakeHttpClient::create(move |mut request| async move {
+ assert_eq!(request.uri().path(), "/completions");
+ let mut body = String::new();
+ request.body_mut().read_to_string(&mut body).await?;
+ let body: serde_json::Value = serde_json::from_str(&body)?;
+ assert_eq!(
+ body["provider_request"].get("max_output_tokens").cloned(),
+ expected.map(|value| json!(value))
+ );
+ let completed = CompletionEvent::Event(json!({
+ "type": "response.completed",
+ "response": {
+ "id": "response-1", "status": "completed", "output": [],
+ "usage": {"input_tokens": 1, "output_tokens": 0, "total_tokens": 1}
+ }
+ }));
+ let ended = CompletionEvent::<serde_json::Value>::Status(
+ CompletionRequestStatus::StreamEnded,
+ );
+ Ok(Response::builder()
+ .status(200)
+ .header(SERVER_SUPPORTS_STATUS_MESSAGES_HEADER_NAME, "true")
+ .body(AsyncBody::from(format!(
+ "{}\n{}\n",
+ serde_json::to_string(&completed)?,
+ serde_json::to_string(&ended)?,
+ )))?)
+ }));
+ let mut request = compact_test_request();
+ request.max_output_tokens = limit;
+ let mut stream = model
+ .stream_completion(request, &cx.to_async())
+ .await
+ .unwrap();
+ while let Some(event) = stream.next().await {
+ event.unwrap();
+ }
+ }
+ }
+
fn compact_test_request() -> LanguageModelRequest {
LanguageModelRequest {
thread_id: Some("thread-123".to_string()),
diff --git a/crates/open_ai/src/completion.rs b/crates/open_ai/src/completion.rs
index cf087070622..37f73bdfd9f 100644
--- a/crates/open_ai/src/completion.rs
+++ b/crates/open_ai/src/completion.rs
@@ -58,6 +58,7 @@ pub fn into_open_ai(
reasoning_effort: Option<ReasoningEffort>,
interleaved_reasoning: bool,
) -> Result<crate::Request> {
+ let max_output_tokens = request.effective_max_output_tokens(max_output_tokens);
if request
.tools
.iter()
@@ -257,6 +258,7 @@ pub fn into_open_ai_response(
supports_none_reasoning_effort: bool,
compaction_state_owner: &LanguageModelProviderId,
) -> Result<ResponseRequest> {
+ let max_output_tokens = request.effective_max_output_tokens(max_output_tokens);
let stream = !model_id.starts_with("o1-");
let LanguageModelRequest {
@@ -272,6 +274,7 @@ pub fn into_open_ai_response(
thinking_effort,
speed,
compact_at_tokens,
+ max_output_tokens: _,
} = request;
let service_tier = service_tier_for(speed);
@@ -1834,6 +1837,7 @@ mod tests {
thinking_effort: Some("high".into()),
speed: None,
compact_at_tokens: None,
+ max_output_tokens: None,
};
let response = into_open_ai_response(
@@ -2020,6 +2024,7 @@ mod tests {
thinking_effort: None,
speed: None,
compact_at_tokens: None,
+ max_output_tokens: None,
};
let response = into_open_ai_response(
@@ -2121,6 +2126,7 @@ mod tests {
thinking_effort: None,
speed: None,
compact_at_tokens: None,
+ max_output_tokens: None,
};
let response = into_open_ai_response(
@@ -2206,6 +2212,7 @@ mod tests {
thinking_effort: None,
speed: None,
compact_at_tokens: None,
+ max_output_tokens: None,
};
let response = into_open_ai_response(
@@ -2272,6 +2279,7 @@ mod tests {
thinking_effort: Some("high".into()),
speed: None,
compact_at_tokens: None,
+ max_output_tokens: None,
};
let response = into_open_ai_response(
@@ -2318,6 +2326,7 @@ mod tests {
thinking_effort: None,
speed,
compact_at_tokens: None,
+ max_output_tokens: None,
};
let response = into_open_ai_response(
@@ -2370,6 +2379,7 @@ mod tests {
thinking_effort: None,
speed,
compact_at_tokens: None,
+ max_output_tokens: None,
};
let chat = into_open_ai(
@@ -2415,22 +2425,60 @@ mod tests {
thinking_effort: None,
speed: None,
compact_at_tokens: None,
+ max_output_tokens: None,
};
- let chat = into_open_ai(
- request,
- "compatible-model",
- false,
- false,
- Some(4096),
- ChatCompletionMaxTokensParameter::MaxTokens,
- None,
- false,
- )?;
+ for (requested, model_maximum, expected) in [
+ (None, None, None),
+ (None, Some(4096), Some(4096)),
+ (Some(1024), Some(4096), Some(1024)),
+ (Some(8192), Some(4096), Some(4096)),
+ (Some(1024), None, Some(1024)),
+ ] {
+ let mut request = request.clone();
+ request.max_output_tokens = requested;
+ for (parameter, field, absent_field) in [
+ (
+ ChatCompletionMaxTokensParameter::MaxCompletionTokens,
+ "max_completion_tokens",
+ "max_tokens",
+ ),
+ (
+ ChatCompletionMaxTokensParameter::MaxTokens,
+ "max_tokens",
+ "max_completion_tokens",
+ ),
+ ] {
+ let chat = into_open_ai(
+ request.clone(),
+ "compatible-model",
+ false,
+ false,
+ model_maximum,
+ parameter,
+ None,
+ false,
+ )?;
+ let serialized = serde_json::to_value(chat)?;
+ assert_eq!(serialized[field].as_u64(), expected);
+ assert!(serialized.get(absent_field).is_none());
+ }
- let serialized = serde_json::to_value(&chat)?;
- assert_eq!(serialized.get("max_completion_tokens"), None);
- assert_eq!(serialized["max_tokens"], json!(4096));
+ let response = into_open_ai_response(
+ request,
+ "gpt-4.1",
+ false,
+ false,
+ model_maximum,
+ None,
+ false,
+ &language_model_core::OPEN_AI_PROVIDER_ID,
+ )?;
+ assert_eq!(
+ serde_json::to_value(response)?["max_output_tokens"].as_u64(),
+ expected
+ );
+ }
Ok(())
}
@@ -2454,6 +2502,7 @@ mod tests {
thinking_effort: Some("high".into()),
speed: None,
compact_at_tokens: None,
+ max_output_tokens: None,
};
let response = into_open_ai_response(
@@ -2495,6 +2544,7 @@ mod tests {
thinking_effort: Some("none".into()),
speed: None,
compact_at_tokens: None,
+ max_output_tokens: None,
};
let response = into_open_ai_response(
@@ -2548,6 +2598,7 @@ mod tests {
thinking_effort: None,
speed: None,
compact_at_tokens: None,
+ max_output_tokens: None,
};
let response = into_open_ai_response(
@@ -2640,6 +2691,7 @@ mod tests {
thinking_effort: None,
speed: None,
compact_at_tokens: None,
+ max_output_tokens: None,
};
let response = into_open_ai_response(
@@ -2730,6 +2782,7 @@ mod tests {
thinking_effort: None,
speed: None,
compact_at_tokens: None,
+ max_output_tokens: None,
};
let response = into_open_ai_response(
@@ -4034,6 +4087,7 @@ mod tests {
thinking_effort: None,
speed: None,
compact_at_tokens: None,
+ max_output_tokens: None,
};
let result = into_open_ai(