Skip to content

armgit / zed

Public Imported from GitHub
0

Code at the speed of thought – Zed is a high-performance, multiplayer code editor from the creators of Atom and Tree-sitter.

agent: Fix compaction being billed as un-cached input tokens (#64938)

tidelyead5672e169f
diff --git a/crates/agent/src/thread.rs b/crates/agent/src/thread.rs
index a13541cb212..6be30909a5f 100644
--- a/crates/agent/src/thread.rs
+++ b/crates/agent/src/thread.rs
@@ -43,9 +43,10 @@ use language_model::{
CompletionIntent, LanguageModel, LanguageModelCompletionError, LanguageModelCompletionEvent,
LanguageModelId, LanguageModelImage, LanguageModelProviderId, LanguageModelRegistry,
LanguageModelRequest, LanguageModelRequestMessage, LanguageModelRequestTool,
- LanguageModelToolResult, LanguageModelToolResultContent, LanguageModelToolUse,
- LanguageModelToolUseId, MessageContent, ProviderErrorCategory, Role, SelectedModel, Speed,
- StopReason, TokenUsage, ZED_CLOUD_PROVIDER_ID,
+ LanguageModelToolChoice, LanguageModelToolResult, LanguageModelToolResultContent,
+ LanguageModelToolUse, LanguageModelToolUseId, MessageContent, PromptCompactionStrategy,
+ ProviderErrorCategory, Role, SelectedModel, Speed, StopReason, TokenUsage,
+ ZED_CLOUD_PROVIDER_ID,
};
use project::{Project, trusted_worktrees::TrustedWorktrees};
use prompt_store::ProjectContext;
@@ -4358,51 +4359,8 @@ impl Thread {
let model = self
.model()
.ok_or_else(|| anyhow!(NoModelConfiguredError))?;
- let sandboxing_enabled = crate::sandboxing::sandboxing_enabled(cx);
let tools = if let Some(turn) = self.running_turn.as_ref() {
- turn.tools
- .iter()
- .map(|(tool_name, tool)| {
- log::trace!("Including tool: {}", tool_name);
- let mut description = tool.description().to_string();
- let mut schema = tool.input_schema();
- // TEMPORARY (sandboxing feature flag): with the flag off,
- // the fetch and create_directory descriptions/schemas must
- // not advertise sandbox-dependent behavior (host grants,
- // out-of-project creation via the `reason` field), since
- // the corresponding runtime paths are disabled. Restore
- // the pre-sandboxing model-facing surface here rather than
- // forking the tools; delete this when the flag is removed
- // again.
- if !sandboxing_enabled {
- if tool_name.as_ref() == FetchTool::NAME {
- description =
- "Fetches a URL and returns the content as Markdown.".to_string();
- } else if tool_name.as_ref() == CreateDirectoryTool::NAME {
- description = "Creates a new directory at the specified path within \
- the project. Returns confirmation that the directory was \
- created.\n\nThis tool creates a directory and all necessary \
- parent directories. It should be used whenever you need to \
- create new directories within the project.\nThe only supported \
- path outside the project is `~/.agents/skills` or a descendant, \
- for global agent skills."
- .to_string();
- if let Some(properties) = schema
- .get_mut("properties")
- .and_then(|value| value.as_object_mut())
- {
- properties.remove("reason");
- }
- }
- }
- LanguageModelRequestTool::function(
- tool_name.to_string(),
- description,
- schema,
- tool.supports_input_streaming(),
- )
- })
- .collect::<Vec<_>>()
+ build_request_tools(&turn.tools, cx)
} else {
Vec::new()
};
@@ -4812,13 +4770,53 @@ impl Thread {
model: &LanguageModel,
cx: &App,
) -> LanguageModelRequest {
+ let uses_thread_model = self.model().is_some_and(|thread_model| {
+ thread_model.provider_id == model.provider_id && thread_model.id == model.id
+ });
+ let preserves_request_prefix = uses_thread_model
+ && model.prompt_compaction_strategy()
+ == PromptCompactionStrategy::PreserveRequestPrefix;
+ let enabled_tools = if preserves_request_prefix
+ && model.supports_tools()
+ && model.supports_tool_choice(LanguageModelToolChoice::None)
+ {
+ self.enabled_tools(cx)
+ } else {
+ BTreeMap::new()
+ };
+ let available_tools = enabled_tools.keys().cloned().collect();
+ let tools = build_request_tools(&enabled_tools, cx);
+ // Keeping the tool definitions preserves the cached prefix, but
+ // compaction can't execute or retain tool calls.
+ let tool_choice = (!tools.is_empty()).then_some(LanguageModelToolChoice::None);
+ let (thinking_allowed, thinking_effort) = if preserves_request_prefix {
+ (
+ self.thinking_enabled || !model.supports_disabling_thinking(),
+ self.thinking_effort.clone(),
+ )
+ } else {
+ (false, None)
+ };
+
let mut request = LanguageModelRequest {
thread_id: Some(self.id.to_string()),
+ prompt_cache_key: None,
prompt_id: Some(self.prompt_id.to_string()),
intent: Some(CompletionIntent::ThreadContextSummarization),
+ messages: self.build_request_messages_until(available_tools, insertion_ix, cx),
+ tools,
+ tool_choice,
+ stop: Vec::new(),
temperature: AgentSettings::temperature_for_model(&model, cx),
- messages: self.build_request_messages_until(Vec::new(), insertion_ix, cx),
- ..Default::default()
+ thinking_allowed,
+ thinking_effort,
+ speed: if preserves_request_prefix {
+ self.speed()
+ } else {
+ None
+ },
+ compact_at_tokens: None,
+ max_output_tokens: None,
};
request.messages.push(LanguageModelRequestMessage {
@@ -4950,6 +4948,55 @@ fn total_input_tokens(usage: language_model::TokenUsage) -> u64 {
.saturating_add(usage.cache_read_input_tokens)
}
+fn build_request_tools(
+ tools: &BTreeMap<SharedString, Arc<dyn AnyAgentTool>>,
+ cx: &App,
+) -> Vec<LanguageModelRequestTool> {
+ let sandboxing_enabled = crate::sandboxing::sandboxing_enabled(cx);
+ tools
+ .iter()
+ .map(|(tool_name, tool)| {
+ log::trace!("Including tool: {}", tool_name);
+ let mut description = tool.description().to_string();
+ let mut schema = tool.input_schema();
+ // TEMPORARY (sandboxing feature flag): with the flag off,
+ // the fetch and create_directory descriptions/schemas must
+ // not advertise sandbox-dependent behavior (host grants,
+ // out-of-project creation via the `reason` field), since
+ // the corresponding runtime paths are disabled. Restore
+ // the pre-sandboxing model-facing surface here rather than
+ // forking the tools; delete this when the flag is removed
+ // again.
+ if !sandboxing_enabled {
+ if tool_name.as_ref() == FetchTool::NAME {
+ description = "Fetches a URL and returns the content as Markdown.".to_string();
+ } else if tool_name.as_ref() == CreateDirectoryTool::NAME {
+ description = "Creates a new directory at the specified path within \
+ the project. Returns confirmation that the directory was \
+ created.\n\nThis tool creates a directory and all necessary \
+ parent directories. It should be used whenever you need to \
+ create new directories within the project.\nThe only supported \
+ path outside the project is `~/.agents/skills` or a descendant, \
+ for global agent skills."
+ .to_string();
+ if let Some(properties) = schema
+ .get_mut("properties")
+ .and_then(|value| value.as_object_mut())
+ {
+ properties.remove("reason");
+ }
+ }
+ }
+ LanguageModelRequestTool::function(
+ tool_name.to_string(),
+ description,
+ schema,
+ tool.supports_input_streaming(),
+ )
+ })
+ .collect()
+}
+
/// Reserves output without subtracting it from an independent input ceiling.
fn compaction_input_capacity(
input_limit: u64,
@@ -8579,6 +8626,216 @@ mod tests {
);
}
+ fn enable_replay_image_tool(thread: &Entity<Thread>, cx: &mut App) {
+ let profile_id = thread.read(cx).profile().clone();
+ let mut settings = AgentSettings::get_global(cx).clone();
+ settings
+ .profiles
+ .get_mut(&profile_id)
+ .expect("thread profile should exist")
+ .tools
+ .insert(ReplayImageTool::NAME.into(), true);
+ AgentSettings::override_global(settings, cx);
+ thread.update(cx, |thread, _cx| thread.add_tool(ReplayImageTool));
+ }
+
+ fn supports_tools_and_tool_choice(model: &mut LanguageModel) {
+ model.supports_tools = true;
+ model.tool_choice_support = language_model::LanguageModelToolChoiceSupport::ALL;
+ }
+
+ #[gpui::test]
+ async fn test_prompt_compaction_uses_default_strategy_without_opt_in(cx: &mut TestAppContext) {
+ let (thread, _event_stream, fake) = setup_thread_for_test(cx).await;
+ let model = fake.update_model("fake", |model| {
+ supports_tools_and_tool_choice(model);
+ model.supports_thinking = true;
+ });
+
+ let request = cx.update(|cx| {
+ enable_replay_image_tool(&thread, cx);
+ thread.update(cx, |thread, cx| {
+ thread.set_model(model.clone(), cx);
+ thread.set_thinking_enabled(true, cx);
+ thread.set_thinking_effort(Some("high".to_string()), cx);
+ thread.set_speed(Speed::Fast, cx);
+ assert_eq!(thread.enabled_tools(cx).len(), 1);
+ thread
+ .messages
+ .push(user_text_message(ClientUserMessageId::new(), "old user"));
+ thread.build_compaction_request(thread.messages.len(), &model, cx)
+ })
+ });
+
+ assert!(request.tools.is_empty());
+ assert_eq!(request.tool_choice, None);
+ assert!(!request.thinking_allowed);
+ assert_eq!(request.thinking_effort, None);
+ assert_eq!(request.speed, None);
+ }
+
+ #[gpui::test]
+ async fn test_prompt_compaction_reuses_request_prefix_when_enabled(cx: &mut TestAppContext) {
+ let (thread, _event_stream, fake) = setup_thread_for_test(cx).await;
+ let model = fake.update_model("fake", |model| {
+ supports_tools_and_tool_choice(model);
+ model.supports_thinking = true;
+ model.prompt_compaction_strategy = PromptCompactionStrategy::PreserveRequestPrefix;
+ });
+
+ cx.update(|cx| {
+ enable_replay_image_tool(&thread, cx);
+ thread.update(cx, |thread, cx| {
+ thread.set_model(model.clone(), cx);
+ thread.set_thinking_enabled(true, cx);
+ thread.set_thinking_effort(Some("high".to_string()), cx);
+ thread.set_speed(Speed::Fast, cx);
+ });
+ });
+
+ let _events = cx
+ .update(|cx| {
+ thread.update(cx, |thread, cx| {
+ thread.send(ClientUserMessageId::new(), vec!["context to compact"], cx)
+ })
+ })
+ .unwrap();
+ cx.run_until_parked();
+
+ let completion_request = fake.pending_completions().pop().unwrap();
+ assert_eq!(completion_request.tools.len(), 1);
+ fake.send_text(&model, &completion_request, "assistant response");
+ fake.end_stream(&model, &completion_request);
+ cx.run_until_parked();
+
+ let _events = cx
+ .update(|cx| {
+ thread.update(cx, |thread, cx| {
+ thread.compact(ClientUserMessageId::new(), cx)
+ })
+ })
+ .unwrap();
+ cx.run_until_parked();
+
+ let compaction_request = fake.pending_completions().pop().unwrap();
+ assert_eq!(
+ compaction_request.intent,
+ Some(CompletionIntent::ThreadContextSummarization)
+ );
+ assert_eq!(compaction_request.tools, completion_request.tools);
+ assert_eq!(
+ compaction_request.tool_choice,
+ Some(LanguageModelToolChoice::None)
+ );
+ assert_eq!(
+ compaction_request.thinking_allowed,
+ completion_request.thinking_allowed
+ );
+ assert_eq!(
+ compaction_request.thinking_effort,
+ completion_request.thinking_effort
+ );
+ assert_eq!(completion_request.speed, Some(Speed::Fast));
+ assert_eq!(compaction_request.speed, completion_request.speed);
+ // The compaction request extends the previous request with the
+ // assistant response and the compaction prompt.
+ assert_eq!(
+ compaction_request.messages.len(),
+ completion_request.messages.len() + 2
+ );
+ for (compaction_message, completion_message) in compaction_request
+ .messages
+ .iter()
+ .zip(&completion_request.messages)
+ {
+ assert_eq!(compaction_message.role, completion_message.role);
+ assert_eq!(compaction_message.content, completion_message.content);
+ assert_eq!(
+ compaction_message.reasoning_details,
+ completion_message.reasoning_details
+ );
+ }
+ }
+
+ #[gpui::test]
+ async fn test_prompt_compaction_omits_tool_choice_without_tools(cx: &mut TestAppContext) {
+ let (thread, _event_stream, fake) = setup_thread_for_test(cx).await;
+ let model = fake.update_model("fake", |model| {
+ supports_tools_and_tool_choice(model);
+ model.prompt_compaction_strategy = PromptCompactionStrategy::PreserveRequestPrefix;
+ });
+
+ let request = cx.update(|cx| {
+ thread.update(cx, |thread, cx| {
+ thread.set_model(model.clone(), cx);
+ thread
+ .messages
+ .push(user_text_message(ClientUserMessageId::new(), "old user"));
+ thread.build_compaction_request(thread.messages.len(), &model, cx)
+ })
+ });
+
+ assert!(request.tools.is_empty());
+ assert_eq!(request.tool_choice, None);
+ }
+
+ #[gpui::test]
+ async fn test_prompt_compaction_omits_tools_without_tool_choice_support(
+ cx: &mut TestAppContext,
+ ) {
+ let (thread, _event_stream, fake) = setup_thread_for_test(cx).await;
+ let model = fake.update_model("fake", |model| {
+ model.supports_tools = true;
+ model.prompt_compaction_strategy = PromptCompactionStrategy::PreserveRequestPrefix;
+ });
+
+ let request = cx.update(|cx| {
+ enable_replay_image_tool(&thread, cx);
+ thread.update(cx, |thread, cx| {
+ thread.set_model(model.clone(), cx);
+ assert_eq!(thread.enabled_tools(cx).len(), 1);
+ thread
+ .messages
+ .push(user_text_message(ClientUserMessageId::new(), "old user"));
+ thread.build_compaction_request(thread.messages.len(), &model, cx)
+ })
+ });
+
+ assert!(request.tools.is_empty());
+ assert_eq!(request.tool_choice, None);
+ }
+
+ #[gpui::test]
+ async fn test_prompt_compaction_rebuilds_prompt_for_different_model(cx: &mut TestAppContext) {
+ let (thread, _event_stream, fake) = setup_thread_for_test(cx).await;
+ let thread_model = fake.update_model("fake", |model| model.supports_thinking = true);
+ let compaction_model = fake.update_model("compaction", |model| {
+ supports_tools_and_tool_choice(model);
+ model.supports_thinking = true;
+ model.prompt_compaction_strategy = PromptCompactionStrategy::PreserveRequestPrefix;
+ });
+
+ let request = cx.update(|cx| {
+ enable_replay_image_tool(&thread, cx);
+ thread.update(cx, |thread, cx| {
+ thread.set_model(thread_model.clone(), cx);
+ thread.set_thinking_enabled(true, cx);
+ thread.set_thinking_effort(Some("high".to_string()), cx);
+ thread.set_speed(Speed::Fast, cx);
+ thread
+ .messages
+ .push(user_text_message(ClientUserMessageId::new(), "old user"));
+ thread.build_compaction_request(thread.messages.len(), &compaction_model, cx)
+ })
+ });
+
+ assert!(request.tools.is_empty());
+ assert_eq!(request.tool_choice, None);
+ assert!(!request.thinking_allowed);
+ assert_eq!(request.thinking_effort, None);
+ assert_eq!(request.speed, None);
+ }
+
/// When `agent.compaction_model` is configured, manual `/compact` streams
/// to the configured model rather than the thread's primary model.
#[gpui::test]
diff --git a/crates/language_model/src/language_model.rs b/crates/language_model/src/language_model.rs
index 7eb4f7254f3..050f07b44c5 100644
--- a/crates/language_model/src/language_model.rs
+++ b/crates/language_model/src/language_model.rs
@@ -139,6 +139,9 @@ pub struct LanguageModel {
pub supports_explicit_compaction_output_limit: bool,
/// The provider-enforced input size required for explicit compaction.
pub minimum_explicit_compaction_input_tokens: Option<u64>,
+ /// How Zed builds prompt-based compaction requests. This does not affect
+ /// provider-native compaction.
+ pub prompt_compaction_strategy: PromptCompactionStrategy,
pub supports_images: bool,
pub supports_tools: bool,
pub tool_choice_support: LanguageModelToolChoiceSupport,
@@ -155,6 +158,16 @@ pub struct LanguageModel {
pub max_output_tokens: Option<u64>,
}
+#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
+pub enum PromptCompactionStrategy {
+ #[default]
+ RebuildPrompt,
+ /// Preserve tools, system instructions, conversation history, and reasoning
+ /// settings while disabling tool calls. Providers must only opt in when
+ /// `tool_choice: none` is enforced without invalidating that cached prefix.
+ PreserveRequestPrefix,
+}
+
/// Which [`LanguageModelToolChoice`] values a model accepts.
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub struct LanguageModelToolChoiceSupport {
@@ -212,6 +225,7 @@ impl LanguageModel {
supports_explicit_compaction: false,
supports_explicit_compaction_output_limit: false,
minimum_explicit_compaction_input_tokens: None,
+ prompt_compaction_strategy: PromptCompactionStrategy::default(),
supports_images: false,
supports_tools: false,
tool_choice_support: LanguageModelToolChoiceSupport::default(),
@@ -333,6 +347,10 @@ impl LanguageModel {
self.minimum_explicit_compaction_input_tokens
}
+ pub fn prompt_compaction_strategy(&self) -> PromptCompactionStrategy {
+ self.prompt_compaction_strategy
+ }
+
pub fn supports_images(&self) -> bool {
self.supports_images
}
diff --git a/crates/language_models/src/provider/deepseek.rs b/crates/language_models/src/provider/deepseek.rs
index a921b1e7752..27ead82e269 100644
--- a/crates/language_models/src/provider/deepseek.rs
+++ b/crates/language_models/src/provider/deepseek.rs
@@ -14,8 +14,8 @@ use language_model::{
LanguageModelProvider, LanguageModelProviderId, LanguageModelProviderName,
LanguageModelProviderState, LanguageModelRequest, LanguageModelToolChoice,
LanguageModelToolChoiceSupport, LanguageModelToolResultContent, LanguageModelToolUse,
- MessageContent, ModelRateLimiters, ProviderSettingsView, RateLimiter, Role, StopReason,
- TokenUsage, env_var, unavailable_error,
+ MessageContent, ModelRateLimiters, PromptCompactionStrategy, ProviderSettingsView, RateLimiter,
+ Role, StopReason, TokenUsage, env_var, unavailable_error,
};
pub use settings::DeepseekAvailableModel as AvailableModel;
use settings::{Settings, SettingsStore};
@@ -330,6 +330,7 @@ fn language_model(model: &deepseek::Model) -> LanguageModel {
Arc::default()
},
tool_choice_support: LanguageModelToolChoiceSupport::ALL,
+ prompt_compaction_strategy: PromptCompactionStrategy::PreserveRequestPrefix,
supports_images: model.supports_images(),
max_output_tokens: model.max_output_tokens(),
..LanguageModel::new(
diff --git a/crates/language_models/src/provider/llama_cpp.rs b/crates/language_models/src/provider/llama_cpp.rs
index fe2caefead3..28962f77e3c 100644
--- a/crates/language_models/src/provider/llama_cpp.rs
+++ b/crates/language_models/src/provider/llama_cpp.rs
@@ -11,8 +11,8 @@ use language_model::{
LanguageModelId, LanguageModelName, LanguageModelProvider, LanguageModelProviderId,
LanguageModelProviderName, LanguageModelProviderState, LanguageModelRequest,
LanguageModelToolChoice, LanguageModelToolChoiceSupport, LanguageModelToolResultContent,
- MessageContent, ModelRateLimiters, ProviderSettingsView, RateLimiter, Role,
- SubPageProviderSettings, env_var, unavailable_error,
+ MessageContent, ModelRateLimiters, PromptCompactionStrategy, ProviderSettingsView, RateLimiter,
+ Role, SubPageProviderSettings, env_var, unavailable_error,
};
use llama_cpp::{
LLAMA_CPP_API_URL, ModelEntry, Props, get_models, get_props, stream_chat_completion,
@@ -570,6 +570,7 @@ impl LlamaCppLanguageModelProvider {
None => LanguageModelName::from(display_name.to_string()),
};
LanguageModel {
+ prompt_compaction_strategy: PromptCompactionStrategy::PreserveRequestPrefix,
supports_tools: capabilities.supports_tools,
tool_choice_support: LanguageModelToolChoiceSupport {
auto: capabilities.supports_tools,
diff --git a/crates/language_models/src/provider/mistral.rs b/crates/language_models/src/provider/mistral.rs
index b7d966481e2..4738c84bf48 100644
--- a/crates/language_models/src/provider/mistral.rs
+++ b/crates/language_models/src/provider/mistral.rs
@@ -12,7 +12,8 @@ use language_model::{
LanguageModelProviderId, LanguageModelProviderName, LanguageModelProviderState,
LanguageModelRequest, LanguageModelToolChoice, LanguageModelToolChoiceSupport,
LanguageModelToolResultContent, LanguageModelToolUse, MessageContent, ModelRateLimiters,
- ProviderSettingsView, RateLimiter, Role, StopReason, TokenUsage, env_var, unavailable_error,
+ PromptCompactionStrategy, ProviderSettingsView, RateLimiter, Role, StopReason, TokenUsage,
+ env_var, unavailable_error,
};
pub use mistral::{MISTRAL_API_URL, StreamResponse};
pub use settings::MistralAvailableModel as AvailableModel;
@@ -327,6 +328,7 @@ fn language_model(model: &mistral::Model) -> LanguageModel {
any: supports_tools,
none: supports_tools,
},
+ prompt_compaction_strategy: PromptCompactionStrategy::PreserveRequestPrefix,
supports_images: model.supports_images(),
supports_thinking: model.supports_thinking(),
supports_disabling_thinking: model.supports_disabling_thinking(),
diff --git a/crates/language_models/src/provider/open_ai.rs b/crates/language_models/src/provider/open_ai.rs
index def85704efa..4179ad10984 100644
--- a/crates/language_models/src/provider/open_ai.rs
+++ b/crates/language_models/src/provider/open_ai.rs
@@ -11,7 +11,8 @@ use language_model::{
LanguageModelId, LanguageModelName, LanguageModelProvider, LanguageModelProviderId,
LanguageModelProviderName, LanguageModelProviderState, LanguageModelRequest,
LanguageModelToolChoiceSupport, ModelRateLimiters, OPEN_AI_PROVIDER_ID, OPEN_AI_PROVIDER_NAME,
- ProviderSettingsView, RateLimiter, env_var, stream_in_background, unavailable_error,
+ PromptCompactionStrategy, ProviderSettingsView, RateLimiter, env_var, stream_in_background,
+ unavailable_error,
};
use open_ai::{
ResponseStreamEvent,
@@ -707,6 +708,7 @@ fn language_model(model: &open_ai::Model) -> LanguageModel {
supports_tools: true,
supports_images,
tool_choice_support: LanguageModelToolChoiceSupport::ALL,
+ prompt_compaction_strategy: PromptCompactionStrategy::PreserveRequestPrefix,
supports_streaming_tools: true,
supports_thinking: supports_selectable_thinking_effort(model),
supports_fast_mode: model.supports_priority(),
diff --git a/crates/language_models/src/provider/open_ai_compatible.rs b/crates/language_models/src/provider/open_ai_compatible.rs
index 971d5b4b1fa..449a5db6435 100644
--- a/crates/language_models/src/provider/open_ai_compatible.rs
+++ b/crates/language_models/src/provider/open_ai_compatible.rs
@@ -9,8 +9,8 @@ use language_model::{
LanguageModelCompletionStream, LanguageModelEffortLevel, LanguageModelId, LanguageModelName,
LanguageModelProvider, LanguageModelProviderId, LanguageModelProviderName,
LanguageModelProviderState, LanguageModelRequest, LanguageModelToolChoiceSupport,
- ModelRateLimiters, ProviderSettingsView, RateLimiter, SubPageProviderSettings,
- unavailable_error,
+ ModelRateLimiters, PromptCompactionStrategy, ProviderSettingsView, RateLimiter,
+ SubPageProviderSettings, unavailable_error,
};
use open_ai::{
ResponseStreamEvent,
@@ -95,6 +95,7 @@ impl OpenAiCompatibleLanguageModelProvider {
any: model.capabilities.tools,
none: true,
},
+ prompt_compaction_strategy: PromptCompactionStrategy::RebuildPrompt,
supports_streaming_tools: true,
supports_thinking: default_thinking_reasoning_effort(model).is_some(),
supported_effort_levels: supported_thinking_effort_levels(model).into(),
diff --git a/crates/language_models/src/provider/x_ai.rs b/crates/language_models/src/provider/x_ai.rs
index 087d201aeeb..3aaf349faf0 100644
--- a/crates/language_models/src/provider/x_ai.rs
+++ b/crates/language_models/src/provider/x_ai.rs
@@ -10,8 +10,8 @@ use language_model::{
LanguageModelClient, LanguageModelCompletionError, LanguageModelCompletionStream,
LanguageModelEffortLevel, LanguageModelId, LanguageModelName, LanguageModelProvider,
LanguageModelProviderId, LanguageModelProviderName, LanguageModelProviderState,
- LanguageModelRequest, LanguageModelToolChoiceSupport, ModelRateLimiters, ProviderSettingsView,
- RateLimiter, env_var, unavailable_error,
+ LanguageModelRequest, LanguageModelToolChoiceSupport, ModelRateLimiters,
+ PromptCompactionStrategy, ProviderSettingsView, RateLimiter, env_var, unavailable_error,
};
pub use settings::XaiAvailableModel as AvailableModel;
use settings::{Settings, SettingsStore};
@@ -395,6 +395,7 @@ fn language_model(model: &x_ai::Model) -> LanguageModel {
supports_images: model.supports_images(),
supports_streaming_tools: true,
tool_choice_support: LanguageModelToolChoiceSupport::ALL,
+ prompt_compaction_strategy: PromptCompactionStrategy::PreserveRequestPrefix,
supports_thinking: model.supports_reasoning_effort(),
supported_effort_levels: supported_thinking_effort_levels(model).into(),
max_output_tokens: model.max_output_tokens(),
diff --git a/crates/language_models_cloud/src/language_models_cloud.rs b/crates/language_models_cloud/src/language_models_cloud.rs
index 3772c08458a..5c2c99bea46 100644
--- a/crates/language_models_cloud/src/language_models_cloud.rs
+++ b/crates/language_models_cloud/src/language_models_cloud.rs
@@ -23,8 +23,9 @@ use language_model::{
LanguageModelCompletionEvent, LanguageModelCompletionStream, LanguageModelEffortLevel,
LanguageModelId, LanguageModelName, LanguageModelProviderId, LanguageModelProviderName,
LanguageModelRequest, LanguageModelToolChoiceSupport, ModelRateLimiters, OPEN_AI_PROVIDER_ID,
- OPEN_AI_PROVIDER_NAME, ProviderErrorCategory, RateLimiter, X_AI_PROVIDER_ID,
- X_AI_PROVIDER_NAME, ZED_CLOUD_PROVIDER_ID, ZED_CLOUD_PROVIDER_NAME, unavailable_error,
+ OPEN_AI_PROVIDER_NAME, PromptCompactionStrategy, ProviderErrorCategory, RateLimiter,
+ X_AI_PROVIDER_ID, X_AI_PROVIDER_NAME, ZED_CLOUD_PROVIDER_ID, ZED_CLOUD_PROVIDER_NAME,
+ unavailable_error,
};
use schemars::JsonSchema;
@@ -546,6 +547,11 @@ pub fn language_model(model: &cloud_llm_client::LanguageModel) -> LanguageModel
none: true,
any: model.provider != Anthropic || anthropic::supports_forced_tool_use(&model.id.0),
},
+ // Anthropic invalidates cached messages when `tool_choice` changes.
+ prompt_compaction_strategy: match model.provider {
+ OpenAi | XAi => PromptCompactionStrategy::PreserveRequestPrefix,
+ Anthropic | Google => PromptCompactionStrategy::RebuildPrompt,
+ },
supports_split_token_display: matches!(model.provider, OpenAi | XAi),
max_output_tokens: Some(model.max_output_tokens as u64),
..LanguageModel::new(
@@ -2140,6 +2146,18 @@ mod tests {
);
}
+ #[test]
+ fn prompt_compaction_preserves_prefix_only_for_supported_upstreams() {
+ assert_eq!(
+ language_model(&cloud_test_config()).prompt_compaction_strategy,
+ PromptCompactionStrategy::PreserveRequestPrefix
+ );
+ assert_eq!(
+ language_model(&cloud_anthropic_test_config()).prompt_compaction_strategy,
+ PromptCompactionStrategy::RebuildPrompt
+ );
+ }
+
#[test]
fn config_resolves_current_model_list_and_rejects_unlisted_models() {
let listed = language_model(&cloud_test_config());