Skip to content

armgit / zed

Public Imported from GitHub
0

Code at the speed of thought – Zed is a high-performance, multiplayer code editor from the creators of Atom and Tree-sitter.

Improve explicit compaction for Anthropic models (#62139)

Eric Holk984bf4d466ed
diff --git a/crates/anthropic/src/anthropic.rs b/crates/anthropic/src/anthropic.rs
index 52f528aaa49..597839446ff 100644
--- a/crates/anthropic/src/anthropic.rs
+++ b/crates/anthropic/src/anthropic.rs
@@ -46,6 +46,9 @@ pub const FABLE_FALLBACK_MODEL_ID: &str = "claude-opus-4-8";
/// <https://platform.claude.com/docs/en/build-with-claude/compaction>
pub const COMPACTION_BETA_HEADER: &str = "compact-2026-01-12";
+/// The smallest input-token trigger Anthropic accepts for compaction.
+pub const MIN_COMPACTION_TRIGGER_TOKENS: u64 = 50_000;
+
#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))]
#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq)]
pub enum AnthropicModelMode {
@@ -187,6 +190,7 @@ impl Model {
| "claude-opus-4-8"
| "claude-opus-4-7"
| "claude-opus-4-6"
+ | "claude-sonnet-5"
| "claude-sonnet-4-6"
);
@@ -855,6 +859,9 @@ pub enum ContextManagementEdit {
Compact {
#[serde(default, skip_serializing_if = "Option::is_none")]
trigger: Option<CompactionTrigger>,
+ /// Stops after emitting the compaction block instead of continuing the response.
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ pause_after_compaction: Option<bool>,
},
}
@@ -902,6 +909,26 @@ pub struct Request {
pub top_p: Option<f32>,
}
+impl Request {
+ /// Configures this request to stop after native compaction.
+ ///
+ /// Tools are removed because Anthropic's internal summarizer may otherwise
+ /// invoke one instead of producing replacement context.
+ pub fn into_compact_request(mut self) -> Self {
+ self.tools.clear();
+ self.tool_choice = None;
+ self.context_management = Some(ContextManagement {
+ edits: vec![ContextManagementEdit::Compact {
+ trigger: Some(CompactionTrigger::InputTokens {
+ value: MIN_COMPACTION_TRIGGER_TOKENS,
+ }),
+ pause_after_compaction: Some(true),
+ }],
+ });
+ self
+ }
+}
+
#[derive(Debug, Default, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum Speed {
@@ -932,14 +959,15 @@ pub struct Usage {
pub cache_creation_input_tokens: Option<u64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub cache_read_input_tokens: Option<u64>,
- /// Only populated when a new compaction is triggered during the request.
+ /// Per-sampling token counts returned when the compaction beta is enabled.
+ ///
/// The top-level token fields exclude compaction iterations, so total
/// billable usage is the sum across all iterations.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub iterations: Option<Vec<UsageIteration>>,
}
-#[derive(Debug, Serialize, Deserialize)]
+#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct UsageIteration {
#[serde(rename = "type")]
pub iteration_type: UsageIterationType,
diff --git a/crates/anthropic/src/completion.rs b/crates/anthropic/src/completion.rs
index 6c0b65114b0..b0ab8d13ec8 100644
--- a/crates/anthropic/src/completion.rs
+++ b/crates/anthropic/src/completion.rs
@@ -1,5 +1,6 @@
use anyhow::{Result, anyhow};
use collections::HashMap;
+use futures::stream::BoxStream;
use futures::{Stream, StreamExt};
use language_model_core::{
CompactedContext, CompactionUpdate, LanguageModelCompletionError, LanguageModelCompletionEvent,
@@ -61,6 +62,62 @@ pub fn provider_compaction_encrypted_content(
Ok(Some(state.payload().into()))
}
+/// Collects one paused compaction stream into replacement context and usage.
+///
+/// # Errors
+///
+/// Returns an error when the stream fails, ends without finalized replacement
+/// context, or reports that the provider abandoned compaction.
+pub async fn collect_compaction_result(
+ mut stream: BoxStream<
+ 'static,
+ Result<LanguageModelCompletionEvent, LanguageModelCompletionError>,
+ >,
+ provider_name: LanguageModelProviderName,
+) -> Result<(CompactedContext, TokenUsage), LanguageModelCompletionError> {
+ let mut context = None;
+ let mut usage = TokenUsage::default();
+ while let Some(event) = stream.next().await {
+ match event? {
+ LanguageModelCompletionEvent::Compaction(CompactionUpdate::Finished(
+ compacted_context,
+ )) => {
+ context = Some(compacted_context);
+ }
+ LanguageModelCompletionEvent::Compaction(CompactionUpdate::Failed) => {
+ return Err(LanguageModelCompletionError::Other(anyhow!(
+ "{provider_name} abandoned compaction without producing replacement context"
+ )));
+ }
+ LanguageModelCompletionEvent::UsageUpdate(updated_usage) => {
+ usage = updated_usage;
+ }
+ LanguageModelCompletionEvent::Stop(_) => {
+ return context.map(|context| (context, usage)).ok_or(
+ LanguageModelCompletionError::StreamEndedUnexpectedly {
+ provider: provider_name,
+ },
+ );
+ }
+ LanguageModelCompletionEvent::Queued { .. }
+ | LanguageModelCompletionEvent::Started
+ | LanguageModelCompletionEvent::Text(_)
+ | LanguageModelCompletionEvent::Thinking { .. }
+ | LanguageModelCompletionEvent::RedactedThinking { .. }
+ | LanguageModelCompletionEvent::ToolUse(_)
+ | LanguageModelCompletionEvent::ToolUseJsonParseError { .. }
+ | LanguageModelCompletionEvent::StartMessage { .. }
+ | LanguageModelCompletionEvent::ReasoningDetails(_)
+ | LanguageModelCompletionEvent::Compaction(CompactionUpdate::Started)
+ | LanguageModelCompletionEvent::Compaction(CompactionUpdate::SummaryDelta(_)) => {}
+ }
+ }
+
+ Err(LanguageModelCompletionError::StreamEndedUnexpectedly {
+ provider: provider_name,
+ })
+}
+
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub enum AnthropicPromptCacheMode {
Disabled,
@@ -414,6 +471,7 @@ pub fn into_anthropic(
context_management: request.compact_at_tokens.map(|value| ContextManagement {
edits: vec![ContextManagementEdit::Compact {
trigger: Some(CompactionTrigger::InputTokens { value }),
+ pause_after_compaction: None,
}],
}),
})
@@ -656,6 +714,7 @@ impl AnthropicEventMapper {
"max_tokens" => StopReason::MaxTokens,
"tool_use" => StopReason::ToolUse,
"refusal" => StopReason::Refusal,
+ "compaction" => StopReason::EndTurn,
_ => {
log::error!("Unexpected anthropic stop_reason: {stop_reason}");
StopReason::EndTurn
@@ -717,9 +776,32 @@ fn update_usage(usage: &mut Usage, new: &Usage) {
if let Some(cache_read_input_tokens) = new.cache_read_input_tokens {
usage.cache_read_input_tokens = Some(cache_read_input_tokens);
}
+ if let Some(iterations) = &new.iterations {
+ usage.iterations = Some(iterations.clone());
+ }
}
fn convert_usage(usage: &Usage) -> TokenUsage {
+ if let Some(iterations) = usage.iterations.as_deref() {
+ return iterations
+ .iter()
+ .fold(TokenUsage::default(), |mut total, iteration| {
+ total.input_tokens = total
+ .input_tokens
+ .saturating_add(iteration.input_tokens.unwrap_or(0));
+ total.output_tokens = total
+ .output_tokens
+ .saturating_add(iteration.output_tokens.unwrap_or(0));
+ total.cache_creation_input_tokens = total
+ .cache_creation_input_tokens
+ .saturating_add(iteration.cache_creation_input_tokens.unwrap_or(0));
+ total.cache_read_input_tokens = total
+ .cache_read_input_tokens
+ .saturating_add(iteration.cache_read_input_tokens.unwrap_or(0));
+ total
+ });
+ }
+
TokenUsage {
input_tokens: usage.input_tokens.unwrap_or(0),
output_tokens: usage.output_tokens.unwrap_or(0),
@@ -732,11 +814,57 @@ fn convert_usage(usage: &Usage) -> TokenUsage {
mod tests {
use super::*;
use crate::{AnthropicModelMode, UsageIteration, UsageIterationType};
+ use futures::executor::block_on;
use language_model_core::{
- ANTHROPIC_PROVIDER_ID, ANTHROPIC_PROVIDER_NAME, LanguageModelImage,
- LanguageModelRequestMessage, MessageContent,
+ ANTHROPIC_PROVIDER_ID, ANTHROPIC_PROVIDER_NAME, LanguageModelCompletionEvent,
+ LanguageModelImage, LanguageModelRequestMessage, MessageContent, TokenUsage,
};
+ #[test]
+ fn test_collect_compaction_result_returns_context_and_usage() {
+ let context = CompactedContext::Summary {
+ content: "Summary of the conversation.".into(),
+ provider_state: None,
+ };
+ let usage = TokenUsage {
+ input_tokens: 60_000,
+ output_tokens: 1_000,
+ ..Default::default()
+ };
+ let stream = futures::stream::iter([
+ Ok(LanguageModelCompletionEvent::Compaction(
+ CompactionUpdate::Started,
+ )),
+ Ok(LanguageModelCompletionEvent::Compaction(
+ CompactionUpdate::Finished(context.clone()),
+ )),
+ Ok(LanguageModelCompletionEvent::UsageUpdate(usage)),
+ Ok(LanguageModelCompletionEvent::Stop(StopReason::EndTurn)),
+ ])
+ .boxed();
+
+ let result = block_on(collect_compaction_result(stream, ANTHROPIC_PROVIDER_NAME)).unwrap();
+
+ assert_eq!(result, (context, usage));
+ }
+
+ #[test]
+ fn test_collect_compaction_result_rejects_abandoned_compaction() {
+ let stream = futures::stream::iter([Ok(LanguageModelCompletionEvent::Compaction(
+ CompactionUpdate::Failed,
+ ))])
+ .boxed();
+
+ let error =
+ block_on(collect_compaction_result(stream, ANTHROPIC_PROVIDER_NAME)).unwrap_err();
+
+ assert!(
+ error
+ .to_string()
+ .contains("abandoned compaction without producing replacement context")
+ );
+ }
+
#[test]
fn test_caching_uses_top_level_auto_and_long_lived_prefix() {
let request = LanguageModelRequest {
@@ -1231,6 +1359,24 @@ mod tests {
);
}
+ #[test]
+ fn test_compact_request_pauses_at_minimum_trigger() {
+ let request =
+ request_with_assistant_content(vec![MessageContent::Text("Response".to_string())])
+ .into_compact_request();
+
+ assert_eq!(
+ serde_json::to_value(&request.context_management).unwrap(),
+ serde_json::json!({
+ "edits": [{
+ "type": "compact_20260112",
+ "trigger": { "type": "input_tokens", "value": 50_000 },
+ "pause_after_compaction": true
+ }]
+ })
+ );
+ }
+
#[test]
fn test_no_context_management_without_compact_at_tokens() {
let result =
@@ -1533,7 +1679,7 @@ mod tests {
}
#[test]
- fn test_usage_iterations_parsed_from_message_delta() {
+ fn test_usage_iterations_aggregated_from_message_delta() {
let event: Event = serde_json::from_value(serde_json::json!({
"type": "message_delta",
"delta": { "stop_reason": "end_turn", "stop_sequence": null },
@@ -1568,5 +1714,13 @@ mod tests {
..
}
));
+ assert_eq!(
+ convert_usage(&usage),
+ TokenUsage {
+ input_tokens: 180_100,
+ output_tokens: 1_239,
+ ..Default::default()
+ }
+ );
}
}
diff --git a/crates/language_model/src/language_model.rs b/crates/language_model/src/language_model.rs
index 5760d071d02..316b09c5a53 100644
--- a/crates/language_model/src/language_model.rs
+++ b/crates/language_model/src/language_model.rs
@@ -145,6 +145,11 @@ pub trait LanguageModel: Send + Sync {
false
}
+ /// The provider-enforced input size required for explicit compaction.
+ fn minimum_explicit_compaction_input_tokens(&self) -> Option<u64> {
+ None
+ }
+
fn compact(
&self,
_request: LanguageModelRequest,
diff --git a/crates/language_models/src/provider/anthropic.rs b/crates/language_models/src/provider/anthropic.rs
index 8edb4998ffd..772deb54f02 100644
--- a/crates/language_models/src/provider/anthropic.rs
+++ b/crates/language_models/src/provider/anthropic.rs
@@ -9,7 +9,7 @@ use gpui::{App, AppContext, AsyncApp, Context, Entity, SharedString, Task};
use http_client::{CustomHeaders, HttpClient};
use language_model::{
ANTHROPIC_PROVIDER_ID, ANTHROPIC_PROVIDER_NAME, ApiKeyConfiguration, ApiKeyState,
- AuthenticateError, EnvVar, FastModeConfirmation, IconOrSvg, LanguageModel,
+ AuthenticateError, CompactionResult, EnvVar, FastModeConfirmation, IconOrSvg, LanguageModel,
LanguageModelCompletionError, LanguageModelCompletionEvent, LanguageModelId, LanguageModelName,
LanguageModelProvider, LanguageModelProviderId, LanguageModelProviderName,
LanguageModelProviderState, LanguageModelRequest, LanguageModelToolChoice,
@@ -19,6 +19,7 @@ use settings::{Settings, SettingsStore};
use std::sync::{Arc, LazyLock};
use ui::IconName;
+use anthropic::completion::collect_compaction_result;
pub use anthropic::completion::{AnthropicEventMapper, AnthropicPromptCacheMode, into_anthropic};
pub use settings::AnthropicAvailableModel as AvailableModel;
@@ -380,6 +381,11 @@ fn available_model_to_anthropic_model(available: &AvailableModel) -> anthropic::
#[cfg(test)]
mod tests {
use super::*;
+ use futures::AsyncReadExt as _;
+ use http_client::{AsyncBody, FakeHttpClient};
+ use language_model::{LanguageModelRequestMessage, MessageContent};
+ use serde_json::json;
+ use std::sync::Mutex;
fn parse_available_model(json: &str) -> AvailableModel {
serde_json::from_str(json).expect("test fixture should parse")
@@ -444,6 +450,232 @@ mod tests {
assert!(!model.supports_adaptive_thinking);
assert!(model.supported_effort_levels.is_empty());
}
+
+ #[gpui::test]
+ fn direct_anthropic_supports_explicit_compaction_after_minimum_input(
+ cx: &mut gpui::TestAppContext,
+ ) {
+ let provider = direct_anthropic_test_provider(FakeHttpClient::with_404_response(), cx);
+ let model = direct_anthropic_test_model(&provider);
+
+ assert!(model.supports_explicit_compaction());
+ assert_eq!(
+ model.minimum_explicit_compaction_input_tokens(),
+ Some(anthropic::MIN_COMPACTION_TRIGGER_TOKENS)
+ );
+ }
+
+ #[gpui::test]
+ async fn direct_anthropic_explicit_compaction_uses_paused_completion(
+ cx: &mut gpui::TestAppContext,
+ ) {
+ let captured_request = Arc::new(Mutex::new(None));
+ let captured_request_for_handler = captured_request.clone();
+ let http_client = FakeHttpClient::create(move |request| {
+ let captured_request = captured_request_for_handler.clone();
+ async move {
+ if request.uri().path() == "/v1/models" {
+ return Ok(http_client::Response::builder()
+ .status(200)
+ .body(AsyncBody::from(r#"{"data":[]}"#))?);
+ }
+
+ let beta_header = request
+ .headers()
+ .get("Anthropic-Beta")
+ .and_then(|value| value.to_str().ok())
+ .map(str::to_string);
+ let mut body = request.into_body();
+ let mut body_text = String::new();
+ body.read_to_string(&mut body_text).await?;
+ *captured_request.lock().unwrap() = Some((beta_header, body_text));
+
+ let response_lines = [
+ json!({
+ "type": "message_start",
+ "message": {
+ "id": "msg_compact",
+ "type": "message",
+ "role": "assistant",
+ "content": [],
+ "model": "claude-opus-4-6",
+ "stop_reason": null,
+ "stop_sequence": null,
+ "usage": {
+ "input_tokens": 0,
+ "output_tokens": 0
+ }
+ }
+ }),
+ json!({
+ "type": "content_block_start",
+ "index": 0,
+ "content_block": {
+ "type": "compaction",
+ "content": null,
+ "encrypted_content": null
+ }
+ }),
+ json!({
+ "type": "content_block_delta",
+ "index": 0,
+ "delta": {
+ "type": "compaction_delta",
+ "content": "Summary of the conversation.",
+ "encrypted_content": "opaque-state"
+ }
+ }),
+ json!({
+ "type": "content_block_stop",
+ "index": 0
+ }),
+ json!({
+ "type": "message_delta",
+ "delta": {
+ "stop_reason": "compaction",
+ "stop_sequence": null
+ },
+ "usage": {
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "iterations": [{
+ "type": "compaction",
+ "input_tokens": 60_000,
+ "output_tokens": 1_000
+ }]
+ }
+ }),
+ json!({"type": "message_stop"}),
+ ]
+ .into_iter()
+ .map(|line| format!("data: {line}"))
+ .collect::<Vec<_>>()
+ .join("\n");
+
+ Ok(http_client::Response::builder()
+ .status(200)
+ .body(AsyncBody::from(format!("{response_lines}\n")))?)
+ }
+ });
+ let provider = direct_anthropic_test_provider(http_client, cx);
+ let store_key = cx.update(|cx| provider.set_api_key(Some("test-key".to_string()), cx));
+ store_key.await.unwrap();
+ let model = direct_anthropic_test_model(&provider);
+ let request = LanguageModelRequest {
+ messages: vec![LanguageModelRequestMessage {
+ role: language_model::Role::User,
+ content: vec![MessageContent::Text("Retain this context.".to_string())],
+ cache: false,
+ reasoning_details: None,
+ }],
+ ..Default::default()
+ };
+
+ let result = model.compact(request, &cx.to_async()).await.unwrap();
+
+ assert_eq!(
+ result.usage,
+ language_model::TokenUsage {
+ input_tokens: 60_000,
+ output_tokens: 1_000,
+ ..Default::default()
+ }
+ );
+ let language_model::CompactedContext::Summary {
+ content,
+ provider_state,
+ } = result.context
+ else {
+ panic!("expected summary compaction");
+ };
+ assert_eq!(content.as_ref(), "Summary of the conversation.");
+ assert_eq!(
+ anthropic::completion::provider_compaction_encrypted_content(
+ &provider_state.expect("expected opaque provider state"),
+ &ANTHROPIC_PROVIDER_ID,
+ )
+ .unwrap()
+ .as_deref(),
+ Some("opaque-state")
+ );
+
+ let (beta_header, body) = captured_request.lock().unwrap().take().unwrap();
+ assert!(
+ beta_header
+ .as_deref()
+ .is_some_and(|header| header.contains(anthropic::COMPACTION_BETA_HEADER))
+ );
+ let body = serde_json::from_str::<serde_json::Value>(&body).unwrap();
+ assert_eq!(
+ body["context_management"],
+ json!({
+ "edits": [{
+ "type": "compact_20260112",
+ "trigger": {
+ "type": "input_tokens",
+ "value": anthropic::MIN_COMPACTION_TRIGGER_TOKENS
+ },
+ "pause_after_compaction": true
+ }]
+ })
+ );
+ assert!(body["tools"].is_null());
+ }
+
+ fn direct_anthropic_test_provider(
+ http_client: Arc<dyn HttpClient>,
+ cx: &mut gpui::TestAppContext,
+ ) -> AnthropicLanguageModelProvider {
+ cx.update(|cx| {
+ let settings_store = SettingsStore::test(cx);
+ cx.set_global(settings_store);
+ AnthropicLanguageModelProvider::new(http_client, Arc::new(TestCredentialsProvider), cx)
+ })
+ }
+
+ fn direct_anthropic_test_model(
+ provider: &AnthropicLanguageModelProvider,
+ ) -> Arc<dyn LanguageModel> {
+ provider.create_language_model(anthropic::Model::from_listed(anthropic::ListModelEntry {
+ id: "claude-opus-4-6".to_string(),
+ display_name: "Claude Opus 4.6".to_string(),
+ max_input_tokens: 1_000_000,
+ max_tokens: 128_000,
+ capabilities: None,
+ }))
+ }
+
+ struct TestCredentialsProvider;
+
+ impl CredentialsProvider for TestCredentialsProvider {
+ fn read_credentials<'a>(
+ &'a self,
+ _url: &'a str,
+ _cx: &'a AsyncApp,
+ ) -> std::pin::Pin<
+ Box<dyn std::future::Future<Output = Result<Option<(String, Vec<u8>)>>> + 'a>,
+ > {
+ Box::pin(async { Ok(None) })
+ }
+
+ fn write_credentials<'a>(
+ &'a self,
+ _url: &'a str,
+ _username: &'a str,
+ _password: &'a [u8],
+ _cx: &'a AsyncApp,
+ ) -> std::pin::Pin<Box<dyn std::future::Future<Output = Result<()>> + 'a>> {
+ Box::pin(async { Ok(()) })
+ }
+
+ fn delete_credentials<'a>(
+ &'a self,
+ _url: &'a str,
+ _cx: &'a AsyncApp,
+ ) -> std::pin::Pin<Box<dyn std::future::Future<Output = Result<()>> + 'a>> {
+ Box::pin(async { Ok(()) })
+ }
+ }
}
pub struct AnthropicModel {
@@ -555,6 +787,54 @@ impl LanguageModel for AnthropicModel {
self.model.supports_compaction
}
+ fn supports_explicit_compaction(&self) -> bool {
+ self.model.supports_compaction
+ }
+
+ fn minimum_explicit_compaction_input_tokens(&self) -> Option<u64> {
+ self.supports_explicit_compaction()
+ .then_some(anthropic::MIN_COMPACTION_TRIGGER_TOKENS)
+ }
+
+ fn compact(
+ &self,
+ request: LanguageModelRequest,
+ cx: &AsyncApp,
+ ) -> BoxFuture<'static, Result<CompactionResult, LanguageModelCompletionError>> {
+ if !self.supports_explicit_compaction() {
+ return async {
+ Err(LanguageModelCompletionError::Other(anyhow::anyhow!(
+ "this Anthropic model does not support explicit compaction"
+ )))
+ }
+ .boxed();
+ }
+
+ let mut request = match into_anthropic(
+ request,
+ self.model.request_id(false).to_string(),
+ self.model.default_temperature,
+ self.model.max_output_tokens,
+ self.model.mode.clone(),
+ AnthropicPromptCacheMode::Automatic,
+ &PROVIDER_ID,
+ ) {
+ Ok(request) => request.into_compact_request(),
+ Err(error) => return async move { Err(error.into()) }.boxed(),
+ };
+ if !self.model.supports_speed {
+ request.speed = None;
+ }
+ let request = self.stream_completion(request, cx);
+ let future = self.request_limiter.run(async move {
+ let response = request.await?;
+ let stream = AnthropicEventMapper::new(PROVIDER_NAME, PROVIDER_ID).map_stream(response);
+ let (context, usage) = collect_compaction_result(stream.boxed(), PROVIDER_NAME).await?;
+ Ok(CompactionResult { context, usage })
+ });
+ future.boxed()
+ }
+
fn supported_effort_levels(&self) -> Vec<language_model::LanguageModelEffortLevel> {
self.model
.supported_effort_levels
diff --git a/crates/language_models_cloud/src/language_models_cloud.rs b/crates/language_models_cloud/src/language_models_cloud.rs
index 695d4324890..e75d0b2536d 100644
--- a/crates/language_models_cloud/src/language_models_cloud.rs
+++ b/crates/language_models_cloud/src/language_models_cloud.rs
@@ -39,7 +39,9 @@ use std::task::Poll;
use std::time::Duration;
use thiserror::Error;
-use anthropic::completion::{AnthropicEventMapper, AnthropicPromptCacheMode, into_anthropic};
+use anthropic::completion::{
+ AnthropicEventMapper, AnthropicPromptCacheMode, collect_compaction_result, into_anthropic,
+};
use google_ai::completion::{GoogleEventMapper, into_google};
use open_ai::completion::{
ChatCompletionMaxTokensParameter, OpenAiEventMapper, OpenAiResponseEventMapper, into_open_ai,
@@ -234,6 +236,73 @@ impl<TP: CloudLlmTokenProvider> CloudLanguageModel<TP> {
}
}
+impl<TP: CloudLlmTokenProvider + 'static> CloudLanguageModel<TP> {
+ fn compact_anthropic(
+ &self,
+ request: LanguageModelRequest,
+ cx: &AsyncApp,
+ ) -> BoxFuture<'static, Result<CompactionResult, LanguageModelCompletionError>> {
+ let thread_id = request.thread_id.clone();
+ let prompt_id = request.prompt_id.clone();
+ let app_version = self.app_version.clone();
+ let mut request = match into_anthropic(
+ request,
+ self.model.id.to_string(),
+ 1.0,
+ self.model.max_output_tokens as u64,
+ AnthropicModelMode::Default,
+ AnthropicPromptCacheMode::Automatic,
+ &ANTHROPIC_PROVIDER_ID,
+ ) {
+ Ok(request) => request.into_compact_request(),
+ Err(error) => return async move { Err(error.into()) }.boxed(),
+ };
+ if !self.model.supports_fast_mode {
+ request.speed = None;
+ }
+
+ let http_client = self.http_client.clone();
+ let token_provider = self.token_provider.clone();
+ let auth_context = token_provider.auth_context(cx);
+ let future = self.request_limiter.run(async move {
+ let PerformLlmCompletionResponse {
+ response,
+ includes_status_messages,
+ } = Self::perform_llm_completion(
+ &http_client,
+ &*token_provider,
+ auth_context,
+ app_version,
+ CompletionBody {
+ thread_id,
+ prompt_id,
+ provider: cloud_llm_client::LanguageModelProvider::Anthropic,
+ model: request.model.clone(),
+ provider_request: serde_json::to_value(&request).map_err(|error| {
+ LanguageModelCompletionError::SerializeRequest {
+ provider: ANTHROPIC_PROVIDER_NAME,
+ error,
+ }
+ })?,
+ },
+ )
+ .await?;
+
+ let mut mapper =
+ AnthropicEventMapper::new(ANTHROPIC_PROVIDER_NAME, ANTHROPIC_PROVIDER_ID);
+ let stream = map_cloud_completion_events(
+ Box::pin(response_lines(response, includes_status_messages)),
+ &ANTHROPIC_PROVIDER_NAME,
+ move |event| mapper.map_event(event),
+ );
+ let (context, usage) =
+ collect_compaction_result(stream, ANTHROPIC_PROVIDER_NAME).await?;
+ Ok(CompactionResult { context, usage })
+ });
+ future.boxed()
+ }
+}
+
fn needs_llm_token_refresh(response: &Response<AsyncBody>) -> bool {
response
.headers()
@@ -326,6 +395,94 @@ impl From<ApiError> for LanguageModelCompletionError {
}
}
+impl<TP: CloudLlmTokenProvider + 'static> CloudLanguageModel<TP> {
+ /// Explicit compaction via OpenAI's dedicated compact operation, proxied
+ /// through the cloud's `/completions/compact` endpoint.
+ fn compact_open_ai(
+ &self,
+ request: LanguageModelRequest,
+ cx: &AsyncApp,
+ ) -> BoxFuture<'static, Result<CompactionResult, LanguageModelCompletionError>> {
+ let thread_id = request.thread_id.clone();
+ let prompt_id = request.prompt_id.clone();
+ let app_version = self.app_version.clone();
+ let model_provider = self.model.provider;
+ let provider_name = provider_name(&self.model.provider);
+ let supports_none_reasoning_effort =
+ self.model.supported_effort_levels.iter().any(|effort| {
+ open_ai::ReasoningEffort::from_str(&effort.value)
+ .is_ok_and(|effort| effort == open_ai::ReasoningEffort::None)
+ });
+ // Cloud proxies to OpenAI's own infrastructure, so the resulting
+ // compaction state is owned by (and interchangeable with) OpenAI
+ // proper, not by the cloud transport.
+ let request = match into_open_ai_response(
+ request,
+ &self.model.id.0,
+ self.model.supports_parallel_tool_calls,
+ true,
+ None,
+ None,
+ supports_none_reasoning_effort,
+ &OPEN_AI_PROVIDER_ID,
+ ) {
+ Ok(request) => request,
+ Err(error) => return async move { Err(error.into()) }.boxed(),
+ };
+ let compact_request = request.into_compact_request();
+ let http_client = self.http_client.clone();
+ let token_provider = self.token_provider.clone();
+ let auth_context = token_provider.auth_context(cx);
+ let future = self.request_limiter.run(async move {
+ let PerformLlmCompletionResponse {
+ response,
+ includes_status_messages,
+ } = Self::perform_llm_compaction(
+ &http_client,
+ &*token_provider,
+ auth_context,
+ app_version,
+ CompletionBody {
+ thread_id,
+ prompt_id,
+ provider: model_provider,
+ model: compact_request.model.clone(),
+ provider_request: serde_json::to_value(compact_request).map_err(|error| {
+ LanguageModelCompletionError::SerializeRequest {
+ provider: provider_name.clone(),
+ error,
+ }
+ })?,
+ },
+ )
+ .await?;
+
+ let events = response_lines::<open_ai::responses::CompactedResponse>(
+ response,
+ includes_status_messages,
+ );
+ futures::pin_mut!(events);
+ while let Some(event) = events.next().await {
+ match event.map_err(|error| error.into_completion_error(provider_name.clone()))? {
+ CompletionEvent::Event(response) => {
+ let usage = token_usage_from_response_usage(&response.usage);
+ let context = response
+ .into_compacted_context(OPEN_AI_PROVIDER_ID)
+ .map_err(LanguageModelCompletionError::Other)?;
+ return Ok(CompactionResult { context, usage });
+ }
+ CompletionEvent::Status(_) => {}
+ }
+ }
+
+ Err(LanguageModelCompletionError::StreamEndedUnexpectedly {
+ provider: provider_name,
+ })
+ });
+ future.boxed()
+ }
+}
+
impl<TP: CloudLlmTokenProvider + 'static> LanguageModel for CloudLanguageModel<TP> {
fn id(&self) -> LanguageModelId {
self.id.clone()
@@ -421,8 +578,17 @@ impl<TP: CloudLlmTokenProvider + 'static> LanguageModel for CloudLanguageModel<T
}
fn supports_explicit_compaction(&self) -> bool {
- self.model.provider == cloud_llm_client::LanguageModelProvider::OpenAi
- && self.model.supports_server_side_compaction
+ matches!(
+ self.model.provider,
+ cloud_llm_client::LanguageModelProvider::OpenAi
+ | cloud_llm_client::LanguageModelProvider::Anthropic
+ ) && self.model.supports_server_side_compaction
+ }
+
+ fn minimum_explicit_compaction_input_tokens(&self) -> Option<u64> {
+ (self.model.provider == cloud_llm_client::LanguageModelProvider::Anthropic
+ && self.supports_explicit_compaction())
+ .then_some(anthropic::MIN_COMPACTION_TRIGGER_TOKENS)
}
fn compact(
@@ -439,83 +605,22 @@ impl<TP: CloudLlmTokenProvider + 'static> LanguageModel for CloudLanguageModel<T
.boxed();
}
- let thread_id = request.thread_id.clone();
- let prompt_id = request.prompt_id.clone();
- let app_version = self.app_version.clone();
- let model_provider = self.model.provider;
- let provider_name = provider_name(&self.model.provider);
- let supports_none_reasoning_effort =
- self.model.supported_effort_levels.iter().any(|effort| {
- open_ai::ReasoningEffort::from_str(&effort.value)
- .is_ok_and(|effort| effort == open_ai::ReasoningEffort::None)
- });
- // Cloud proxies to OpenAI's own infrastructure, so the resulting
- // compaction state is owned by (and interchangeable with) OpenAI
- // proper, not by the cloud transport.
- let request = match into_open_ai_response(
- request,
- &self.model.id.0,
- self.model.supports_parallel_tool_calls,
- true,
- None,
- None,
- supports_none_reasoning_effort,
- &OPEN_AI_PROVIDER_ID,
- ) {
- Ok(request) => request,
- Err(error) => return async move { Err(error.into()) }.boxed(),
- };
- let compact_request = request.into_compact_request();
- let http_client = self.http_client.clone();
- let token_provider = self.token_provider.clone();
- let auth_context = token_provider.auth_context(cx);
- let future = self.request_limiter.run(async move {
- let PerformLlmCompletionResponse {
- response,
- includes_status_messages,
- } = Self::perform_llm_compaction(
- &http_client,
- &*token_provider,
- auth_context,
- app_version,
- CompletionBody {
- thread_id,
- prompt_id,
- provider: model_provider,
- model: compact_request.model.clone(),
- provider_request: serde_json::to_value(compact_request).map_err(|error| {
- LanguageModelCompletionError::SerializeRequest {
- provider: provider_name.clone(),
- error,
- }
- })?,
- },
- )
- .await?;
-
- let events = response_lines::<open_ai::responses::CompactedResponse>(
- response,
- includes_status_messages,
- );
- futures::pin_mut!(events);
- while let Some(event) = events.next().await {
- match event.map_err(|error| error.into_completion_error(provider_name.clone()))? {
- CompletionEvent::Event(response) => {
- let usage = token_usage_from_response_usage(&response.usage);
- let context = response
- .into_compacted_context(OPEN_AI_PROVIDER_ID)
- .map_err(LanguageModelCompletionError::Other)?;
- return Ok(CompactionResult { context, usage });
- }
- CompletionEvent::Status(_) => {}
- }
+ match self.model.provider {
+ cloud_llm_client::LanguageModelProvider::OpenAi => self.compact_open_ai(request, cx),
+ cloud_llm_client::LanguageModelProvider::Anthropic => {
+ self.compact_anthropic(request, cx)
}
-
- Err(LanguageModelCompletionError::StreamEndedUnexpectedly {
- provider: provider_name,
- })
- });
- future.boxed()
+ // Unreachable while the `supports_explicit_compaction` guard
+ // above holds, but a provider mismatch should degrade to the
+ // same unsupported error rather than panic.
+ cloud_llm_client::LanguageModelProvider::Google
+ | cloud_llm_client::LanguageModelProvider::XAi => async {
+ Err(LanguageModelCompletionError::Other(anyhow::anyhow!(
+ "this cloud model does not support explicit compaction"
+ )))
+ }
+ .boxed(),
+ }
}
fn supported_effort_levels(&self) -> Vec<LanguageModelEffortLevel> {
@@ -1240,6 +1345,151 @@ mod tests {
);
}
+ #[gpui::test]
+ async fn cloud_anthropic_explicit_compaction_uses_paused_completion(
+ cx: &mut gpui::TestAppContext,
+ ) {
+ let captured_request = Arc::new(Mutex::new(None));
+ let captured_request_for_handler = captured_request.clone();
+ let http_client = FakeHttpClient::create(move |request| {
+ let captured_request = captured_request_for_handler.clone();
+ async move {
+ let uri = request.uri().to_string();
+ let mut body = request.into_body();
+ let mut body_text = String::new();
+ body.read_to_string(&mut body_text).await?;
+ *captured_request.lock().unwrap() = Some((uri, body_text));
+
+ let response_lines = [
+ json!({
+ "event": {
+ "type": "message_start",
+ "message": {
+ "id": "msg_compact",
+ "type": "message",
+ "role": "assistant",
+ "content": [],
+ "model": "claude-opus-4-6",
+ "stop_reason": null,
+ "stop_sequence": null,
+ "usage": {
+ "input_tokens": 0,
+ "output_tokens": 0
+ }
+ }
+ }
+ }),
+ json!({
+ "event": {
+ "type": "content_block_start",
+ "index": 0,
+ "content_block": {
+ "type": "compaction",
+ "content": null,
+ "encrypted_content": null
+ }
+ }
+ }),
+ json!({
+ "event": {
+ "type": "content_block_delta",
+ "index": 0,
+ "delta": {
+ "type": "compaction_delta",
+ "content": "Summary of the conversation.",
+ "encrypted_content": "opaque-state"
+ }
+ }
+ }),
+ json!({
+ "event": {
+ "type": "content_block_stop",
+ "index": 0
+ }
+ }),
+ json!({
+ "event": {
+ "type": "message_delta",
+ "delta": {
+ "stop_reason": "compaction",
+ "stop_sequence": null
+ },
+ "usage": {
+ "input_tokens": 0,
+ "output_tokens": 0,
+ "iterations": [{
+ "type": "compaction",
+ "input_tokens": 60_000,
+ "output_tokens": 1_000
+ }]
+ }
+ }
+ }),
+ json!({"event": {"type": "message_stop"}}),
+ ]
+ .into_iter()
+ .map(|line| line["event"].to_string())
+ .collect::<Vec<_>>()
+ .join("\n");
+
+ Ok(http_client::Response::builder()
+ .status(200)
+ .body(AsyncBody::from(format!("{response_lines}\n")))?)
+ }
+ });
+ let model = cloud_anthropic_test_model(http_client);
+
+ let result = model
+ .compact(compact_test_request(), &cx.to_async())
+ .await
+ .unwrap();
+
+ assert_eq!(
+ result.usage,
+ language_model::TokenUsage {
+ input_tokens: 60_000,
+ output_tokens: 1_000,
+ ..Default::default()
+ }
+ );
+ let language_model::CompactedContext::Summary {
+ content,
+ provider_state,
+ } = result.context
+ else {
+ panic!("expected summary compaction");
+ };
+ assert_eq!(content.as_ref(), "Summary of the conversation.");
+ assert_eq!(
+ anthropic::completion::provider_compaction_encrypted_content(
+ &provider_state.expect("expected opaque provider state"),
+ &ANTHROPIC_PROVIDER_ID,
+ )
+ .unwrap()
+ .as_deref(),
+ Some("opaque-state")
+ );
+
+ let (uri, body) = captured_request.lock().unwrap().take().unwrap();
+ assert_eq!(uri, "http://test.example/completions?");
+ let body = serde_json::from_str::<serde_json::Value>(&body).unwrap();
+ assert_eq!(body["provider"], "anthropic");
+ assert_eq!(
+ body["provider_request"]["context_management"],
+ json!({
+ "edits": [{
+ "type": "compact_20260112",
+ "trigger": {
+ "type": "input_tokens",
+ "value": anthropic::MIN_COMPACTION_TRIGGER_TOKENS
+ },
+ "pause_after_compaction": true
+ }]
+ })
+ );
+ assert!(body["provider_request"]["tools"].is_null());
+ }
+
#[gpui::test]
async fn cloud_explicit_compaction_rejects_output_without_compaction_item(
cx: &mut gpui::TestAppContext,
@@ -1455,6 +1705,17 @@ mod tests {
);
}
+ #[test]
+ fn cloud_anthropic_supports_explicit_compaction_after_minimum_input() {
+ let model = cloud_anthropic_test_model(FakeHttpClient::with_404_response());
+
+ assert!(model.supports_explicit_compaction());
+ assert_eq!(
+ model.minimum_explicit_compaction_input_tokens(),
+ Some(anthropic::MIN_COMPACTION_TRIGGER_TOKENS)
+ );
+ }
+
fn compact_test_request() -> LanguageModelRequest {
LanguageModelRequest {
thread_id: Some("thread-123".to_string()),
@@ -1469,6 +1730,38 @@ mod tests {
}
}
+ fn cloud_anthropic_test_model(
+ http_client: Arc<HttpClientWithUrl>,
+ ) -> CloudLanguageModel<TestTokenProvider> {
+ CloudLanguageModel {
+ id: LanguageModelId::from("claude-opus-4-6".to_string()),
+ model: Arc::new(cloud_llm_client::LanguageModel {
+ provider: cloud_llm_client::LanguageModelProvider::Anthropic,
+ id: cloud_llm_client::LanguageModelId(Arc::from("claude-opus-4-6")),
+ display_name: "Claude Opus 4.6".to_string(),
+ is_latest: true,
+ max_token_count: 1_000_000,
+ max_token_count_in_max_mode: None,
+ max_output_tokens: 128_000,
+ supports_tools: true,
+ supports_images: true,
+ supports_thinking: true,
+ supports_disabling_thinking: true,
+ supports_fast_mode: true,
+ supports_server_side_compaction: true,
+ supported_effort_levels: Vec::new(),
+ supports_streaming_tools: true,
+ supports_parallel_tool_calls: false,
+ is_disabled: false,
+ disabled_reason: None,
+ }),
+ token_provider: Arc::new(TestTokenProvider),
+ http_client,
+ app_version: None,
+ request_limiter: RateLimiter::new(4),
+ }
+ }
+
fn cloud_test_model(
http_client: Arc<HttpClientWithUrl>,
) -> CloudLanguageModel<TestTokenProvider> {