Code at the speed of thought – Zed is a high-performance, multiplayer code editor from the creators of Atom and Tree-sitter.
use anthropic::AnthropicModelMode;
use anyhow::{Context as _, Result};
use cloud_api_client::ClientApiError;
use cloud_llm_client::{
CLIENT_SUPPORTS_STATUS_MESSAGES_HEADER_NAME, CLIENT_SUPPORTS_STATUS_STREAM_ENDED_HEADER_NAME,
CLIENT_SUPPORTS_X_AI_HEADER_NAME, CompletionBody, CompletionEvent, CompletionRequestStatus,
EXPIRED_LLM_TOKEN_HEADER_NAME, ListModelsResponse, OUTDATED_LLM_TOKEN_HEADER_NAME,
SERVER_SUPPORTS_STATUS_MESSAGES_HEADER_NAME, ZED_VERSION_HEADER_NAME,
};
use futures::{
AsyncBufReadExt, AsyncReadExt as _, FutureExt, Stream, StreamExt, future::BoxFuture,
io::BufReader, stream,
};
use google_ai::GoogleModelMode;
use gpui::{App, AppContext, Context, Task};
use http_client::http::{HeaderMap, HeaderValue};
use http_client::{
AsyncBody, HttpClient, HttpClientWithUrl, HttpRequestExt, Method, Response, StatusCode,
};
use language_model::{
ANTHROPIC_PROVIDER_ID, ANTHROPIC_PROVIDER_NAME, CompactionResult, DisabledReason,
GOOGLE_PROVIDER_ID, GOOGLE_PROVIDER_NAME, LanguageModel, LanguageModelCompletionError,
LanguageModelCompletionEvent, LanguageModelCompletionStream, LanguageModelEffortLevel,
LanguageModelId, LanguageModelName, LanguageModelProviderId, LanguageModelProviderName,
LanguageModelRequest, LanguageModelToolChoiceSupport, ModelRateLimiters, OPEN_AI_PROVIDER_ID,
OPEN_AI_PROVIDER_NAME, PromptCompactionStrategy, ProviderErrorCategory, RateLimiter,
X_AI_PROVIDER_ID, X_AI_PROVIDER_NAME, ZED_CLOUD_PROVIDER_ID, ZED_CLOUD_PROVIDER_NAME,
unavailable_error,
};
use schemars::JsonSchema;
use semver::Version;
use serde::{Deserialize, Serialize, de::DeserializeOwned};
use std::collections::VecDeque;
use std::pin::Pin;
use std::str::FromStr;
use std::sync::Arc;
use std::task::Poll;
use std::time::Duration;
use thiserror::Error;
use anthropic::completion::{
AnthropicEventMapper, AnthropicPromptCacheMode, collect_compaction_result, into_anthropic,
};
use google_ai::completion::{GoogleEventMapper, into_google};
use language_model::chat_completion::ChatCompletionEventMapper;
use open_ai::completion::{
ChatCompletionMaxTokensParameter, OpenAiResponseEventMapper, into_open_ai,
into_open_ai_response, token_usage_from_response_usage,
};
const PROVIDER_ID: LanguageModelProviderId = ZED_CLOUD_PROVIDER_ID;
const PROVIDER_NAME: LanguageModelProviderName = ZED_CLOUD_PROVIDER_NAME;
/// Trait for acquiring and refreshing LLM authentication tokens.
pub trait CloudLlmTokenProvider: Send + Sync {
type AuthContext: Clone + Send + 'static;
fn auth_context(&self, cx: &impl AppContext) -> Self::AuthContext;
fn cached_token(
&self,
auth_context: Self::AuthContext,
) -> BoxFuture<'static, Result<String, ClientApiError>>;
fn refresh_token(
&self,
auth_context: Self::AuthContext,
) -> BoxFuture<'static, Result<String, ClientApiError>>;
/// Whether the user has consented to upstream providers retaining
/// inference logs for models that require it (see
/// [`LanguageModel::requires_data_retention`]).
fn has_data_retention_consent(&self, cx: &impl AppContext) -> bool;
}
/// Why an authenticated request to the Zed LLM service failed before a
/// response arrived.
#[derive(Debug, Error)]
pub enum LlmRequestError {
/// The token provider could not supply an LLM token.
#[error("failed to acquire an LLM token")]
Token(#[source] ClientApiError),
#[error("failed to build the LLM request")]
BuildRequest(#[source] anyhow::Error),
#[error("failed to send the LLM request")]
Send(#[source] anyhow::Error),
}
impl LlmRequestError {
/// Classifies the failure as a completion error for a request to `host`.
///
/// Credential and billing failures while acquiring a token become provider
/// rejections, so callers can tell them apart from connectivity problems.
pub fn into_completion_error(self, host: String) -> LanguageModelCompletionError {
match self {
Self::Token(ClientApiError::Unauthorized) => {
LanguageModelCompletionError::from_provider_response(
PROVIDER_NAME,
Some(StatusCode::UNAUTHORIZED),
None,
"Zed rejected the credentials for this request.".to_string(),
None,
ProviderErrorCategory::Authentication,
)
}
Self::Token(ClientApiError::NotSignedIn) => {
LanguageModelCompletionError::from_provider_response(
PROVIDER_NAME,
None,
None,
"Not signed in to Zed.".to_string(),
None,
ProviderErrorCategory::Authentication,
)
}
Self::Token(ClientApiError::ServerError { status, .. })
if status == StatusCode::PAYMENT_REQUIRED =>
{
payment_required_error(status)
}
Self::Token(ClientApiError::ServerError { status, body, .. }) => {
LanguageModelCompletionError::from_http_status(PROVIDER_NAME, status, body, None)
}
Self::Token(error @ ClientApiError::ConnectionFailed { .. }) => {
LanguageModelCompletionError::HttpSend {
provider: PROVIDER_NAME,
host,
error: error.into(),
}
}
Self::Send(error) => LanguageModelCompletionError::HttpSend {
provider: PROVIDER_NAME,
host,
error,
},
error @ (Self::Token(
ClientApiError::InvalidResponse(_) | ClientApiError::RequestBuildFailed(_),
)
| Self::BuildRequest(_)) => LanguageModelCompletionError::Other(error.into()),
}
}
}
fn payment_required_error(status: StatusCode) -> LanguageModelCompletionError {
LanguageModelCompletionError::from_provider_response(
PROVIDER_NAME,
Some(status),
None,
"payment required to use this language model; please upgrade your account".to_string(),
None,
ProviderErrorCategory::PaymentRequired,
)
}
/// Sends an authenticated request to the Zed LLM service, retrying once with
/// a refreshed token if the server signals that the cached LLM token is
/// expired or otherwise rejected. Returns the raw response so callers can
/// inspect headers and stream the body.
pub async fn authenticated_llm_request<TP: CloudLlmTokenProvider>(
http_client: &HttpClientWithUrl,
token_provider: &TP,
auth_context: TP::AuthContext,
build_request: impl Fn(&str) -> Result<http_client::Request<AsyncBody>>,
) -> Result<Response<AsyncBody>, LlmRequestError> {
let send = async |token: String| {
let request = build_request(&token).map_err(LlmRequestError::BuildRequest)?;
http_client
.send(request)
.await
.map_err(LlmRequestError::Send)
};
let token = token_provider
.cached_token(auth_context.clone())
.await
.map_err(LlmRequestError::Token)?;
let response = send(token).await?;
if !needs_llm_token_refresh(&response) && response.status() != StatusCode::UNAUTHORIZED {
return Ok(response);
}
log::info!("LLM token rejected; refreshing and retrying request");
let token = token_provider
.refresh_token(auth_context)
.await
.map_err(LlmRequestError::Token)?;
send(token).await
}
#[derive(Default, Clone, Debug, PartialEq, Serialize, Deserialize, JsonSchema)]
#[serde(tag = "type", rename_all = "lowercase")]
pub enum ModelMode {
#[default]
Default,
Thinking {
/// The maximum number of tokens to use for reasoning. Must be lower than the model's `max_output_tokens`.
budget_tokens: Option<u32>,
},
}
impl From<ModelMode> for AnthropicModelMode {
fn from(value: ModelMode) -> Self {
match value {
ModelMode::Default => AnthropicModelMode::Default,
ModelMode::Thinking { budget_tokens } => AnthropicModelMode::Thinking { budget_tokens },
}
}
}
pub struct PerformLlmCompletionResponse {
pub response: Response<AsyncBody>,
pub includes_status_messages: bool,
}
enum StatusMessages {
Request,
Omit,
}
impl<TP: CloudLlmTokenProvider> CloudModelProvider<TP> {
pub async fn perform_llm_completion(
http_client: &HttpClientWithUrl,
token_provider: &TP,
auth_context: TP::AuthContext,
app_version: Option<Version>,
body: CompletionBody,
) -> Result<PerformLlmCompletionResponse, LanguageModelCompletionError> {
Self::perform_llm_request(
"/completions",
StatusMessages::Request,
http_client,
token_provider,
auth_context,
app_version,
body,
)
.await
}
async fn perform_llm_compaction(
http_client: &HttpClientWithUrl,
token_provider: &TP,
auth_context: TP::AuthContext,
app_version: Option<Version>,
body: CompletionBody,
) -> Result<PerformLlmCompletionResponse, LanguageModelCompletionError> {
Self::perform_llm_request(
"/completions/compact",
StatusMessages::Request,
http_client,
token_provider,
auth_context,
app_version,
body,
)
.await
}
async fn perform_llm_request(
path: &str,
status_messages: StatusMessages,
http_client: &HttpClientWithUrl,
token_provider: &TP,
auth_context: TP::AuthContext,
app_version: Option<Version>,
body: CompletionBody,
) -> Result<PerformLlmCompletionResponse, LanguageModelCompletionError> {
let url = http_client
.build_zed_llm_url(path, &[])
.map_err(LanguageModelCompletionError::Other)?;
let host = url.host_str().unwrap_or(url.as_str()).to_owned();
let body = serde_json::to_string(&body).map_err(|error| {
LanguageModelCompletionError::SerializeRequest {
provider: PROVIDER_NAME,
error,
}
})?;
let mut response =
authenticated_llm_request(http_client, token_provider, auth_context, |token| {
let mut request = http_client::Request::builder()
.method(Method::POST)
.uri(url.as_ref())
.when_some(app_version.as_ref(), |builder, app_version| {
builder.header(ZED_VERSION_HEADER_NAME, app_version.to_string())
})
.header("Content-Type", "application/json")
.header("Authorization", format!("Bearer {token}"));
if matches!(status_messages, StatusMessages::Request) {
request = request
.header(CLIENT_SUPPORTS_STATUS_MESSAGES_HEADER_NAME, "true")
.header(CLIENT_SUPPORTS_STATUS_STREAM_ENDED_HEADER_NAME, "true");
}
Ok(request.body(body.clone().into())?)
})
.await
.map_err(|error| error.into_completion_error(host))?;
let status = response.status();
if status.is_success() {
let includes_status_messages = matches!(status_messages, StatusMessages::Request)
&& response
.headers()
.get(SERVER_SUPPORTS_STATUS_MESSAGES_HEADER_NAME)
.is_some();
return Ok(PerformLlmCompletionResponse {
response,
includes_status_messages,
});
}
if status == StatusCode::PAYMENT_REQUIRED {
return Err(payment_required_error(status));
}
let mut body = String::new();
let headers = response.headers().clone();
response
.body_mut()
.read_to_string(&mut body)
.await
.map_err(|error| LanguageModelCompletionError::ApiReadResponseError {
provider: PROVIDER_NAME,
error,
})?;
Err(ApiError {
status,
body,
headers,
}
.into())
}
}
fn anthropic_request(
config: &cloud_llm_client::LanguageModel,
request: LanguageModelRequest,
) -> Result<anthropic::Request> {
let enable_thinking = request.thinking_allowed && config.supports_thinking;
let effort = request
.thinking_effort
.as_ref()
.and_then(|effort| anthropic::Effort::from_str(effort).ok());
let mut request = into_anthropic(
request,
config.id.to_string(),
1.0,
config.max_output_tokens as u64,
if enable_thinking && effort.is_some() {
AnthropicModelMode::AdaptiveThinking
} else if enable_thinking {
AnthropicModelMode::Thinking {
budget_tokens: Some(4_096),
}
} else {
AnthropicModelMode::Default
},
AnthropicPromptCacheMode::Automatic,
&ANTHROPIC_PROVIDER_ID,
)?;
if !config.supports_fast_mode {
request.speed = None;
}
Ok(request)
}
fn open_ai_request(
config: &cloud_llm_client::LanguageModel,
request: LanguageModelRequest,
) -> Result<open_ai::responses::Request> {
let enable_thinking = request.thinking_allowed && config.supports_thinking;
let effort = request
.thinking_effort
.as_ref()
.and_then(|effort| open_ai::ReasoningEffort::from_str(effort).ok())
.filter(|effort| *effort != open_ai::ReasoningEffort::None);
let supports_none_reasoning_effort = config.supported_effort_levels.iter().any(|effort| {
open_ai::ReasoningEffort::from_str(&effort.value)
.is_ok_and(|effort| effort == open_ai::ReasoningEffort::None)
});
let mut request = into_open_ai_response(
request,
&config.id.0,
config.supports_parallel_tool_calls,
true,
None,
None,
supports_none_reasoning_effort,
&OPEN_AI_PROVIDER_ID,
)?;
if enable_thinking && let Some(effort) = effort {
request.reasoning = Some(open_ai::responses::ReasoningConfig {
effort,
summary: Some(open_ai::responses::ReasoningSummaryMode::Auto),
});
}
Ok(request)
}
fn needs_llm_token_refresh(response: &Response<AsyncBody>) -> bool {
response
.headers()
.get(EXPIRED_LLM_TOKEN_HEADER_NAME)
.is_some()
|| response
.headers()
.get(OUTDATED_LLM_TOKEN_HEADER_NAME)
.is_some()
}
#[derive(Debug, Error)]
#[error("cloud language model request failed with status {status}: {body}")]
struct ApiError {
status: StatusCode,
body: String,
headers: HeaderMap<HeaderValue>,
}
/// Represents error responses from Zed's cloud API.
///
/// Example JSON for an upstream HTTP error:
/// ```json
/// {
/// "code": "upstream_http_error",
/// "message": "Received an error from the Anthropic API: upstream connect error or disconnect/reset before headers, reset reason: connection timeout",
/// "upstream_status": 503
/// }
/// ```
#[derive(Debug, serde::Deserialize)]
struct CloudApiError {
code: String,
message: String,
#[serde(default)]
#[serde(deserialize_with = "deserialize_optional_status_code")]
upstream_status: Option<StatusCode>,
#[serde(default)]
retry_after: Option<f64>,
}
fn deserialize_optional_status_code<'de, D>(deserializer: D) -> Result<Option<StatusCode>, D::Error>
where
D: serde::Deserializer<'de>,
{
let opt: Option<u16> = Option::deserialize(deserializer)?;
Ok(opt.and_then(|code| StatusCode::from_u16(code).ok()))
}
impl From<ApiError> for LanguageModelCompletionError {
fn from(error: ApiError) -> Self {
if let Ok(cloud_error) = serde_json::from_str::<CloudApiError>(&error.body) {
if cloud_error.code.starts_with("upstream_http_") {
let status = if let Some(status) = cloud_error.upstream_status {
status
} else if cloud_error.code.ends_with("_error") {
error.status
} else {
// If there's a status code in the code string (e.g. "upstream_http_429")
// then use that; otherwise, see if the JSON contains a status code.
cloud_error
.code
.strip_prefix("upstream_http_")
.and_then(|code_str| code_str.parse::<u16>().ok())
.and_then(|code| StatusCode::from_u16(code).ok())
.unwrap_or(error.status)
};
let category =
ProviderErrorCategory::from_http_status(status, &cloud_error.message);
return LanguageModelCompletionError::from_provider_response(
PROVIDER_NAME,
Some(status),
Some(cloud_error.code),
cloud_error.message,
cloud_error.retry_after.map(Duration::from_secs_f64),
category,
);
}
let category =
ProviderErrorCategory::from_http_status(error.status, &cloud_error.message);
return LanguageModelCompletionError::from_provider_response(
PROVIDER_NAME,
Some(error.status),
Some(cloud_error.code),
cloud_error.message,
None,
category,
);
}
let retry_after = None;
LanguageModelCompletionError::from_http_status(
PROVIDER_NAME,
error.status,
error.body,
retry_after,
)
}
}
/// Describes a model from the Zed cloud model list.
pub fn language_model(model: &cloud_llm_client::LanguageModel) -> LanguageModel {
use cloud_llm_client::LanguageModelProvider::*;
let supports_explicit_compaction = supports_explicit_compaction(model);
LanguageModel {
upstream_provider_id: Some(match model.provider {
Anthropic => ANTHROPIC_PROVIDER_ID,
OpenAi => OPEN_AI_PROVIDER_ID,
Google => GOOGLE_PROVIDER_ID,
XAi => X_AI_PROVIDER_ID,
}),
upstream_provider_name: Some(match model.provider {
Anthropic => ANTHROPIC_PROVIDER_NAME,
OpenAi => OPEN_AI_PROVIDER_NAME,
Google => GOOGLE_PROVIDER_NAME,
XAi => X_AI_PROVIDER_NAME,
}),
is_latest: model.is_latest,
disabled_reason: if model.is_disabled {
model.disabled_reason.clone().map(DisabledReason::new)
} else {
None
},
requires_data_retention: requires_data_retention(model),
refusal_fallback_model_id: if model.id.0.starts_with(anthropic::FABLE_MODEL_ID_PREFIX) {
Some(anthropic::FABLE_FALLBACK_MODEL_ID)
} else {
None
},
supports_tools: model.supports_tools,
supports_images: model.supports_images,
supports_thinking: model.supports_thinking,
supports_disabling_thinking: model.supports_disabling_thinking,
supports_fast_mode: model.supports_fast_mode,
supports_server_side_compaction: model.supports_server_side_compaction,
supports_explicit_compaction,
supports_explicit_compaction_output_limit: model.provider == Anthropic
&& supports_explicit_compaction,
minimum_explicit_compaction_input_tokens: (model.provider == Anthropic
&& supports_explicit_compaction)
.then_some(anthropic::MIN_COMPACTION_TRIGGER_TOKENS),
supported_effort_levels: model
.supported_effort_levels
.iter()
.map(|effort_level| LanguageModelEffortLevel {
name: effort_level.name.clone().into(),
value: effort_level.value.clone().into(),
is_default: effort_level.is_default.unwrap_or(false),
supports_disabling_thinking: effort_level.supports_disabling_thinking,
})
.collect(),
supports_streaming_tools: model.supports_streaming_tools,
tool_choice_support: LanguageModelToolChoiceSupport {
auto: true,
none: true,
any: model.provider != Anthropic || anthropic::supports_forced_tool_use(&model.id.0),
},
// Anthropic invalidates cached messages when `tool_choice` changes.
prompt_compaction_strategy: match model.provider {
OpenAi | XAi => PromptCompactionStrategy::PreserveRequestPrefix,
Anthropic | Google => PromptCompactionStrategy::RebuildPrompt,
},
supports_split_token_display: matches!(model.provider, OpenAi | XAi),
max_output_tokens: Some(model.max_output_tokens as u64),
..LanguageModel::new(
LanguageModelId::from(model.id.0.to_string()),
LanguageModelName::from(model.display_name.clone()),
PROVIDER_ID,
PROVIDER_NAME,
format!("zed.dev/{}", model.id),
model.max_token_count as u64,
)
}
}
fn requires_data_retention(model: &cloud_llm_client::LanguageModel) -> bool {
// Anthropic cannot offer Fable models with Zero Data Retention
model.id.0.starts_with(anthropic::FABLE_MODEL_ID_PREFIX)
}
fn supports_explicit_compaction(model: &cloud_llm_client::LanguageModel) -> bool {
matches!(
model.provider,
cloud_llm_client::LanguageModelProvider::OpenAi
| cloud_llm_client::LanguageModelProvider::Anthropic
) && model.supports_server_side_compaction
}
pub struct CloudModelProvider<TP: CloudLlmTokenProvider> {
token_provider: Arc<TP>,
http_client: Arc<HttpClientWithUrl>,
app_version: Option<Version>,
models: Vec<Arc<cloud_llm_client::LanguageModel>>,
default_model: Option<Arc<cloud_llm_client::LanguageModel>>,
default_fast_model: Option<Arc<cloud_llm_client::LanguageModel>>,
recommended_models: Vec<Arc<cloud_llm_client::LanguageModel>>,
request_limiters: ModelRateLimiters,
}
impl<TP: CloudLlmTokenProvider + 'static> CloudModelProvider<TP> {
pub fn new(
token_provider: Arc<TP>,
http_client: Arc<HttpClientWithUrl>,
app_version: Option<Version>,
) -> Self {
Self {
token_provider,
http_client,
app_version,
models: Vec::new(),
default_model: None,
default_fast_model: None,
recommended_models: Vec::new(),
request_limiters: ModelRateLimiters::default(),
}
}
pub fn refresh_models(&self, cx: &mut Context<Self>) -> Task<Result<()>> {
let http_client = self.http_client.clone();
let token_provider = self.token_provider.clone();
cx.spawn(async move |this, cx| {
let auth_context = token_provider.auth_context(cx);
let response =
Self::fetch_models_request(&http_client, &*token_provider, auth_context).await?;
this.update(cx, |this, cx| {
this.update_models(response);
cx.notify();
})
})
}
async fn fetch_models_request(
http_client: &HttpClientWithUrl,
token_provider: &TP,
auth_context: TP::AuthContext,
) -> Result<ListModelsResponse> {
let url = http_client.build_zed_llm_url("/models", &[])?;
let mut response =
authenticated_llm_request(http_client, token_provider, auth_context, |token| {
Ok(http_client::Request::builder()
.method(Method::GET)
.header(CLIENT_SUPPORTS_X_AI_HEADER_NAME, "true")
.uri(url.as_ref())
.header("Authorization", format!("Bearer {token}"))
.body(AsyncBody::empty())?)
})
.await
.context("failed to send list models request")?;
if response.status().is_success() {
let mut body = String::new();
response.body_mut().read_to_string(&mut body).await?;
Ok(serde_json::from_str(&body)?)
} else {
let mut body = String::new();
response.body_mut().read_to_string(&mut body).await?;
anyhow::bail!(
"error listing models.\nStatus: {:?}\nBody: {body}",
response.status(),
);
}
}
pub fn update_models(&mut self, response: ListModelsResponse) {
let models: Vec<_> = response.models.into_iter().map(Arc::new).collect();
self.default_model = models
.iter()
.find(|model| {
response
.default_model
.as_ref()
.is_some_and(|default_model_id| &model.id == default_model_id)
})
.cloned();
self.default_fast_model = models
.iter()
.find(|model| {
response
.default_fast_model
.as_ref()
.is_some_and(|default_fast_model_id| &model.id == default_fast_model_id)
})
.cloned();
self.recommended_models = response
.recommended_models
.iter()
.filter_map(|id| models.iter().find(|model| &model.id == id))
.cloned()
.collect();
self.models = models;
}
pub fn clear_models(&mut self) {
self.models.clear();
self.default_model = None;
self.default_fast_model = None;
self.recommended_models.clear();
}
/// The current configuration of `model`, if the model list still contains it.
fn config(
&self,
model: &LanguageModel,
) -> Result<Arc<cloud_llm_client::LanguageModel>, LanguageModelCompletionError> {
self.models
.iter()
.find(|config| config.id.0.as_ref() == model.id.0.as_ref())
.cloned()
.ok_or_else(|| unavailable_error(model))
}
fn check_data_retention_consent(
&self,
config: &cloud_llm_client::LanguageModel,
cx: &App,
) -> Result<(), LanguageModelCompletionError> {
if requires_data_retention(config) && !self.token_provider.has_data_retention_consent(cx) {
return Err(LanguageModelCompletionError::DataRetentionConsentRequired {
model_name: config.display_name.clone(),
});
}
Ok(())
}
fn compact_anthropic(
&self,
config: &cloud_llm_client::LanguageModel,
request_limiter: &RateLimiter,
request: LanguageModelRequest,
cx: &App,
) -> BoxFuture<'static, Result<CompactionResult, LanguageModelCompletionError>> {
let thread_id = request.thread_id.clone();
let prompt_id = request.prompt_id.clone();
let app_version = self.app_version.clone();
let mut request = match into_anthropic(
request,
config.id.to_string(),
1.0,
config.max_output_tokens as u64,
AnthropicModelMode::Default,
AnthropicPromptCacheMode::Automatic,
&ANTHROPIC_PROVIDER_ID,
) {
Ok(request) => request.into_compact_request(),
Err(error) => return async move { Err(error.into()) }.boxed(),
};
if !config.supports_fast_mode {
request.speed = None;
}
let http_client = self.http_client.clone();
let token_provider = self.token_provider.clone();
let auth_context = token_provider.auth_context(cx);
let executor = cx.background_executor().clone();
let future = request_limiter.run(async move {
let PerformLlmCompletionResponse {
response,
includes_status_messages,
} = Self::perform_llm_completion(
&http_client,
&*token_provider,
auth_context,
app_version,
CompletionBody {
thread_id,
prompt_id,
provider: cloud_llm_client::LanguageModelProvider::Anthropic,
model: request.model.clone(),
provider_request: serde_json::to_value(&request).map_err(|error| {
LanguageModelCompletionError::SerializeRequest {
provider: ANTHROPIC_PROVIDER_NAME,
error,
}
})?,
},
)
.await?;
let mut mapper =
AnthropicEventMapper::new(ANTHROPIC_PROVIDER_NAME, ANTHROPIC_PROVIDER_ID);
let stream = map_cloud_completion_events(
Box::pin(response_lines(response, includes_status_messages)),
&ANTHROPIC_PROVIDER_NAME,
move |event| mapper.map_event(event),
);
let stream = language_model::stream_in_background(stream.boxed(), executor);
let (context, usage) =
collect_compaction_result(stream, ANTHROPIC_PROVIDER_NAME).await?;
Ok(CompactionResult { context, usage })
});
future.boxed()
}
/// Explicit compaction via OpenAI's dedicated compact operation, proxied
/// through the cloud's `/completions/compact` endpoint.
fn compact_open_ai(
&self,
config: &cloud_llm_client::LanguageModel,
request_limiter: &RateLimiter,
request: LanguageModelRequest,
cx: &App,
) -> BoxFuture<'static, Result<CompactionResult, LanguageModelCompletionError>> {
let thread_id = request.thread_id.clone();
let prompt_id = request.prompt_id.clone();
let app_version = self.app_version.clone();
let model_provider = config.provider;
let provider_name = provider_name(&config.provider);
let supports_none_reasoning_effort = config.supported_effort_levels.iter().any(|effort| {
open_ai::ReasoningEffort::from_str(&effort.value)
.is_ok_and(|effort| effort == open_ai::ReasoningEffort::None)
});
// Cloud proxies to OpenAI's own infrastructure, so the resulting
// compaction state is owned by (and interchangeable with) OpenAI
// proper, not by the cloud transport.
let request = match into_open_ai_response(
request,
&config.id.0,
config.supports_parallel_tool_calls,
true,
None,
None,
supports_none_reasoning_effort,
&OPEN_AI_PROVIDER_ID,
) {
Ok(request) => request,
Err(error) => return async move { Err(error.into()) }.boxed(),
};
let compact_request = request.into_compact_request();
let http_client = self.http_client.clone();
let token_provider = self.token_provider.clone();
let auth_context = token_provider.auth_context(cx);
let future = request_limiter.run(async move {
let PerformLlmCompletionResponse {
response,
includes_status_messages,
} = Self::perform_llm_compaction(
&http_client,
&*token_provider,
auth_context,
app_version,
CompletionBody {
thread_id,
prompt_id,
provider: model_provider,
model: compact_request.model.clone(),
provider_request: serde_json::to_value(compact_request).map_err(|error| {
LanguageModelCompletionError::SerializeRequest {
provider: provider_name.clone(),
error,
}
})?,
},
)
.await?;
let events = response_lines::<open_ai::responses::CompactedResponse>(
response,
includes_status_messages,
);
futures::pin_mut!(events);
while let Some(event) = events.next().await {
match event.map_err(|error| error.into_completion_error(provider_name.clone()))? {
CompletionEvent::Event(response) => {
let usage = token_usage_from_response_usage(&response.usage);
let context = response
.into_compacted_context(OPEN_AI_PROVIDER_ID)
.map_err(LanguageModelCompletionError::Other)?;
return Ok(CompactionResult { context, usage });
}
CompletionEvent::Status(CompletionRequestStatus::StreamEnded) => break,
CompletionEvent::Status(CompletionRequestStatus::Failed {
code,
message,
retry_after,
..
}) => {
return Err(LanguageModelCompletionError::from_cloud_failure(
provider_name,
code,
message,
retry_after.map(Duration::from_secs_f64),
));
}
CompletionEvent::Status(_) => {}
}
}
Err(LanguageModelCompletionError::StreamEndedUnexpectedly {
provider: provider_name,
})
});
future.boxed()
}
/// Compacts `request`'s conversation with the listed model `model`.
pub fn compact(
&self,
model: &LanguageModel,
request: LanguageModelRequest,
cx: &App,
) -> BoxFuture<'static, Result<CompactionResult, LanguageModelCompletionError>> {
let config = match self.config(model) {
Ok(config) => config,
Err(error) => return async move { Err(error) }.boxed(),
};
let request_limiter = self.request_limiters.for_model(&model.id);
if let Err(error) = self.check_data_retention_consent(&config, cx) {
return async move { Err(error) }.boxed();
}
if !supports_explicit_compaction(&config) {
return async {
Err(LanguageModelCompletionError::Other(anyhow::anyhow!(
"this cloud model does not support explicit compaction"
)))
}
.boxed();
}
match config.provider {
cloud_llm_client::LanguageModelProvider::OpenAi => {
self.compact_open_ai(&config, &request_limiter, request, cx)
}
cloud_llm_client::LanguageModelProvider::Anthropic => {
self.compact_anthropic(&config, &request_limiter, request, cx)
}
// Unreachable while the `supports_explicit_compaction` guard
// above holds, but a provider mismatch should degrade to the
// same unsupported error rather than panic.
cloud_llm_client::LanguageModelProvider::Google
| cloud_llm_client::LanguageModelProvider::XAi => async {
Err(LanguageModelCompletionError::Other(anyhow::anyhow!(
"this cloud model does not support explicit compaction"
)))
}
.boxed(),
}
}
/// Counts `request`'s input tokens for the listed model `model`, or
/// returns `None` when its upstream provider cannot count them.
pub fn count_input_tokens(
&self,
model: &LanguageModel,
request: LanguageModelRequest,
cx: &App,
) -> BoxFuture<'static, Result<Option<u64>, LanguageModelCompletionError>> {
let config = match self.config(model) {
Ok(config) => config,
Err(error) => return async move { Err(error) }.boxed(),
};
let request_limiter = self.request_limiters.for_model(&model.id);
if let Err(error) = self.check_data_retention_consent(&config, cx) {
return async move { Err(error) }.boxed();
}
use cloud_llm_client::LanguageModelProvider;
let provider_request = match config.provider {
LanguageModelProvider::Anthropic => anthropic_request(&config, request)
.and_then(|request| Ok(serde_json::to_value(request)?)),
LanguageModelProvider::OpenAi => open_ai_request(&config, request)
.and_then(|request| Ok(serde_json::to_value(request)?)),
_ => return async { Ok(None) }.boxed(),
};
let body = match provider_request {
Ok(provider_request) => CompletionBody {
thread_id: None,
prompt_id: None,
provider: config.provider,
model: config.id.to_string(),
provider_request,
},
Err(error) => return async move { Err(error.into()) }.boxed(),
};
let http_client = self.http_client.clone();
let token_provider = self.token_provider.clone();
let auth_context = token_provider.auth_context(cx);
let app_version = self.app_version.clone();
request_limiter
.run(async move {
let mut response = Self::perform_llm_request(
"/count_tokens",
StatusMessages::Omit,
&http_client,
&*token_provider,
auth_context,
app_version,
body,
)
.await?
.response;
let mut body = String::new();
response
.body_mut()
.read_to_string(&mut body)
.await
.map_err(|error| LanguageModelCompletionError::ApiReadResponseError {
provider: PROVIDER_NAME,
error,
})?;
#[derive(Deserialize)]
struct CountResponse {
tokens: u64,
}
let count: CountResponse = serde_json::from_str(&body).map_err(|error| {
LanguageModelCompletionError::DeserializeResponse {
provider: PROVIDER_NAME,
error,
}
})?;
Ok(Some(count.tokens))
})
.boxed()
}
/// Streams a completion of `request` from the listed model `model`.
pub fn stream_completion(
&self,
model: &LanguageModel,
request: LanguageModelRequest,
cx: &App,
) -> BoxFuture<'static, Result<LanguageModelCompletionStream, LanguageModelCompletionError>>
{
let config = match self.config(model) {
Ok(config) => config,
Err(error) => return async move { Err(error) }.boxed(),
};
let request_limiter = self.request_limiters.for_model(&model.id);
if let Err(error) = self.check_data_retention_consent(&config, cx) {
return async move { Err(error) }.boxed();
}
let mut request = request;
if request.max_output_tokens.is_some() {
request.max_output_tokens =
request.effective_max_output_tokens(Some(config.max_output_tokens as u64));
}
let thread_id = request.thread_id.clone();
let prompt_id = request.prompt_id.clone();
let app_version = self.app_version.clone();
let provider_name = provider_name(&config.provider);
match config.provider {
cloud_llm_client::LanguageModelProvider::Anthropic => {
let request = match anthropic_request(&config, request) {
Ok(request) => request,
Err(error) => return async move { Err(error.into()) }.boxed(),
};
let http_client = self.http_client.clone();
let token_provider = self.token_provider.clone();
let auth_context = token_provider.auth_context(cx);
let executor = cx.background_executor().clone();
let future = request_limiter.stream(async move {
let PerformLlmCompletionResponse {
response,
includes_status_messages,
} = Self::perform_llm_completion(
&http_client,
&*token_provider,
auth_context,
app_version,
CompletionBody {
thread_id,
prompt_id,
provider: cloud_llm_client::LanguageModelProvider::Anthropic,
model: request.model.clone(),
provider_request: serde_json::to_value(&request).map_err(|error| {
LanguageModelCompletionError::SerializeRequest {
provider: provider_name.clone(),
error,
}
})?,
},
)
.await?;
let mut mapper =
AnthropicEventMapper::new(provider_name.clone(), ANTHROPIC_PROVIDER_ID);
let events = map_cloud_completion_events(
Box::pin(response_lines(response, includes_status_messages)),
&provider_name,
move |event| mapper.map_event(event),
);
Ok(language_model::stream_in_background(
events.boxed(),
executor,
))
});
async move { Ok(future.await?.boxed()) }.boxed()
}
cloud_llm_client::LanguageModelProvider::OpenAi => {
let http_client = self.http_client.clone();
let token_provider = self.token_provider.clone();
let request = match open_ai_request(&config, request) {
Ok(request) => request,
Err(error) => return async move { Err(error.into()) }.boxed(),
};
let auth_context = token_provider.auth_context(cx);
let executor = cx.background_executor().clone();
let future = request_limiter.stream(async move {
let PerformLlmCompletionResponse {
response,
includes_status_messages,
} = Self::perform_llm_completion(
&http_client,
&*token_provider,
auth_context,
app_version,
CompletionBody {
thread_id,
prompt_id,
provider: cloud_llm_client::LanguageModelProvider::OpenAi,
model: request.model.clone(),
provider_request: serde_json::to_value(&request).map_err(|error| {
LanguageModelCompletionError::SerializeRequest {
provider: provider_name.clone(),
error,
}
})?,
},
)
.await?;
let mut mapper = OpenAiResponseEventMapper::new(OPEN_AI_PROVIDER_ID);
let events = map_cloud_completion_events(
Box::pin(response_lines(response, includes_status_messages)),
&provider_name,
move |event| mapper.map_event(event),
);
Ok(language_model::stream_in_background(
events.boxed(),
executor,
))
});
async move { Ok(future.await?.boxed()) }.boxed()
}
cloud_llm_client::LanguageModelProvider::XAi => {
let http_client = self.http_client.clone();
let token_provider = self.token_provider.clone();
let request = match into_open_ai(
request,
&config.id.0,
config.supports_parallel_tool_calls,
false,
None,
ChatCompletionMaxTokensParameter::MaxCompletionTokens,
None,
false,
) {
Ok(request) => request,
Err(error) => return async move { Err(error.into()) }.boxed(),
};
let auth_context = token_provider.auth_context(cx);
let executor = cx.background_executor().clone();
let future = request_limiter.stream(async move {
let PerformLlmCompletionResponse {
response,
includes_status_messages,
} = Self::perform_llm_completion(
&http_client,
&*token_provider,
auth_context,
app_version,
CompletionBody {
thread_id,
prompt_id,
provider: cloud_llm_client::LanguageModelProvider::XAi,
model: request.model.clone(),
provider_request: serde_json::to_value(&request).map_err(|error| {
LanguageModelCompletionError::SerializeRequest {
provider: provider_name.clone(),
error,
}
})?,
},
)
.await?;
let mut mapper = ChatCompletionEventMapper::new();
let events = map_cloud_completion_events(
Box::pin(response_lines(response, includes_status_messages)),
&provider_name,
move |event| mapper.map_event(event),
);
Ok(language_model::stream_in_background(
events.boxed(),
executor,
))
});
async move { Ok(future.await?.boxed()) }.boxed()
}
cloud_llm_client::LanguageModelProvider::Google => {
let http_client = self.http_client.clone();
let token_provider = self.token_provider.clone();
let mode = if config.supports_thinking {
GoogleModelMode::Thinking {
budget_tokens: None,
}
} else {
GoogleModelMode::Default
};
let request = match into_google(request, config.id.to_string(), mode) {
Ok(request) => request,
Err(error) => return async move { Err(error.into()) }.boxed(),
};
let auth_context = token_provider.auth_context(cx);
let future = request_limiter.stream(async move {
let PerformLlmCompletionResponse {
response,
includes_status_messages,
} = Self::perform_llm_completion(
&http_client,
&*token_provider,
auth_context,
app_version,
CompletionBody {
thread_id,
prompt_id,
provider: cloud_llm_client::LanguageModelProvider::Google,
model: request.model.model_id.clone(),
provider_request: serde_json::to_value(&request).map_err(|error| {
LanguageModelCompletionError::SerializeRequest {
provider: provider_name.clone(),
error,
}
})?,
},
)
.await?;
let mut mapper = GoogleEventMapper::new();
Ok(map_cloud_completion_events(
Box::pin(response_lines(response, includes_status_messages)),
&provider_name,
move |event| mapper.map_event(event),
))
});
async move { Ok(future.await?.boxed()) }.boxed()
}
}
}
pub fn models(&self) -> &[Arc<cloud_llm_client::LanguageModel>] {
&self.models
}
pub fn default_model(&self) -> Option<&Arc<cloud_llm_client::LanguageModel>> {
self.default_model.as_ref()
}
pub fn default_fast_model(&self) -> Option<&Arc<cloud_llm_client::LanguageModel>> {
self.default_fast_model.as_ref()
}
pub fn recommended_models(&self) -> &[Arc<cloud_llm_client::LanguageModel>] {
&self.recommended_models
}
}
pub fn map_cloud_completion_events<T, F>(
stream: Pin<Box<dyn Stream<Item = Result<CompletionEvent<T>, ResponseStreamError>> + Send>>,
provider: &LanguageModelProviderName,
mut map_callback: F,
) -> LanguageModelCompletionStream
where
T: DeserializeOwned + 'static,
F: FnMut(T) -> Vec<Result<LanguageModelCompletionEvent, LanguageModelCompletionError>>
+ Send
+ 'static,
{
let provider = provider.clone();
let mut stream = stream.fuse();
let mut saw_stream_ended = false;
let mut done = false;
let mut pending = VecDeque::new();
stream::poll_fn(move |cx| {
loop {
if let Some(item) = pending.pop_front() {
return Poll::Ready(Some(item));
}
if done {
return Poll::Ready(None);
}
match stream.poll_next_unpin(cx) {
Poll::Ready(Some(event)) => {
let items = match event {
Err(error) => {
vec![Err(error.into_completion_error(provider.clone()))]
}
Ok(CompletionEvent::Status(CompletionRequestStatus::StreamEnded)) => {
saw_stream_ended = true;
vec![]
}
Ok(CompletionEvent::Status(status)) => {
LanguageModelCompletionEvent::from_completion_request_status(
status,
provider.clone(),
)
.transpose()
.map(|event| vec![event])
.unwrap_or_default()
}
Ok(CompletionEvent::Event(event)) => map_callback(event),
};
pending.extend(items);
}
Poll::Ready(None) => {
done = true;
if !saw_stream_ended {
return Poll::Ready(Some(Err(
LanguageModelCompletionError::StreamEndedUnexpectedly {
provider: provider.clone(),
},
)));
}
}
Poll::Pending => return Poll::Pending,
}
}
})
.boxed()
}
pub fn provider_name(
provider: &cloud_llm_client::LanguageModelProvider,
) -> LanguageModelProviderName {
match provider {
cloud_llm_client::LanguageModelProvider::Anthropic => ANTHROPIC_PROVIDER_NAME,
cloud_llm_client::LanguageModelProvider::OpenAi => OPEN_AI_PROVIDER_NAME,
cloud_llm_client::LanguageModelProvider::Google => GOOGLE_PROVIDER_NAME,
cloud_llm_client::LanguageModelProvider::XAi => X_AI_PROVIDER_NAME,
}
}
/// A failure while reading the streamed completion response body.
///
/// Kept as a typed error (rather than `anyhow::Error`) so the consumer can
/// attach the provider name and build a structured
/// [`LanguageModelCompletionError`] without a runtime downcast.
pub enum ResponseStreamError {
Read(std::io::Error),
Deserialize(serde_json::Error),
}
impl ResponseStreamError {
fn into_completion_error(
self,
provider: LanguageModelProviderName,
) -> LanguageModelCompletionError {
match self {
ResponseStreamError::Read(error) => {
LanguageModelCompletionError::ApiReadResponseError { provider, error }
}
ResponseStreamError::Deserialize(error) => {
LanguageModelCompletionError::DeserializeResponse { provider, error }
}
}
}
}
pub fn response_lines<T: DeserializeOwned>(
response: Response<AsyncBody>,
includes_status_messages: bool,
) -> impl Stream<Item = Result<CompletionEvent<T>, ResponseStreamError>> {
futures::stream::try_unfold(
(String::new(), BufReader::new(response.into_body())),
move |(mut line, mut body)| async move {
match body.read_line(&mut line).await {
Ok(0) => Ok(None),
Ok(_) => {
let event = if includes_status_messages {
serde_json::from_str::<CompletionEvent<T>>(&line)
.map_err(ResponseStreamError::Deserialize)?
} else {
CompletionEvent::Event(
serde_json::from_str::<T>(&line)
.map_err(ResponseStreamError::Deserialize)?,
)
};
line.clear();
Ok(Some((event, (line, body))))
}
Err(error) => Err(ResponseStreamError::Read(error)),
}
},
)
}
#[cfg(test)]
mod tests {
use super::*;
use futures::TryStreamExt as _;
use http_client::FakeHttpClient;
use http_client::http::{HeaderMap, StatusCode};
use language_model::{
LanguageModelCompletionError, LanguageModelRequestMessage, MessageContent,
ProviderErrorCategory, Role, Speed, StopReason,
};
use serde_json::json;
use std::sync::Mutex;
use std::sync::atomic::{AtomicBool, Ordering};
#[gpui::test]
async fn cloud_google_completion_emits_stop(cx: &mut gpui::TestAppContext) {
for (finish_reason, expected_stop) in [
("STOP", StopReason::EndTurn),
("MAX_TOKENS", StopReason::MaxTokens),
("SAFETY", StopReason::Refusal),
] {
let (provider, model) = cloud_google_test_model(vec![
json!({"event": {"candidates": [{
"content": {"role": "model", "parts": [{"text": "Hello"}]}
}]}}),
json!({"status": CompletionRequestStatus::KeepAlive}),
json!({"event": {
"candidates": [{
"content": {"role": "model", "parts": [{"text": " world"}]},
"finishReason": finish_reason
}],
"usageMetadata": {"promptTokenCount": 10, "candidatesTokenCount": 2}
}}),
json!({"status": "stream_ended"}),
]);
let mut stream = cx
.update(|cx| {
provider.stream_completion(&model, LanguageModelRequest::default(), cx)
})
.await
.unwrap();
assert!(matches!(
stream.next().await,
Some(Ok(LanguageModelCompletionEvent::Text(text))) if text == "Hello"
));
assert!(matches!(
stream.next().await,
Some(Ok(LanguageModelCompletionEvent::UsageUpdate(usage)))
if usage.input_tokens == 10 && usage.output_tokens == 2
));
assert!(matches!(
stream.next().await,
Some(Ok(LanguageModelCompletionEvent::Text(text))) if text == " world"
));
assert!(matches!(
stream.next().await,
Some(Ok(LanguageModelCompletionEvent::Stop(reason))) if reason == expected_stop
));
assert!(stream.next().await.is_none());
}
}
#[gpui::test]
async fn cloud_google_tool_completion_preserves_transport_errors(
cx: &mut gpui::TestAppContext,
) {
for stream_ended in [false, true] {
let mut events = vec![
json!({"event": {"candidates": [{
"content": {"role": "model", "parts": [{
"functionCall": {"name": "list_directory", "args": {"path": "."}}
}]},
"finishReason": "STOP"
}]}}),
json!({"status": CompletionRequestStatus::KeepAlive}),
];
if stream_ended {
events.push(json!({"status": "stream_ended"}));
}
let (provider, model) = cloud_google_test_model(events);
let mut stream = cx
.update(|cx| {
provider.stream_completion(&model, LanguageModelRequest::default(), cx)
})
.await
.unwrap();
assert!(matches!(
stream.next().await,
Some(Ok(LanguageModelCompletionEvent::ToolUse(tool)))
if tool.name.as_ref() == "list_directory"
));
assert!(matches!(
stream.next().await,
Some(Ok(LanguageModelCompletionEvent::Stop(StopReason::ToolUse)))
));
if !stream_ended {
assert!(matches!(
stream.next().await,
Some(Err(
LanguageModelCompletionError::StreamEndedUnexpectedly { .. }
))
));
}
assert!(stream.next().await.is_none());
}
}
#[gpui::test]
async fn cloud_google_incomplete_completion_does_not_emit_stop(cx: &mut gpui::TestAppContext) {
for stream_ended in [false, true] {
let mut events = vec![json!({"event": {"candidates": [{
"content": {"role": "model", "parts": [{"text": "Partial answer"}]}
}]}})];
if stream_ended {
events.push(json!({"status": "stream_ended"}));
}
let (provider, model) = cloud_google_test_model(events);
let mut stream = cx
.update(|cx| {
provider.stream_completion(&model, LanguageModelRequest::default(), cx)
})
.await
.unwrap();
assert!(matches!(
stream.next().await,
Some(Ok(LanguageModelCompletionEvent::Text(text))) if text == "Partial answer"
));
if !stream_ended {
assert!(matches!(
stream.next().await,
Some(Err(
LanguageModelCompletionError::StreamEndedUnexpectedly { .. }
))
));
}
assert!(stream.next().await.is_none());
}
}
#[gpui::test]
async fn cloud_explicit_compaction_forwards_supported_request_fields(
cx: &mut gpui::TestAppContext,
) {
let language_model = language_model(&cloud_test_config());
assert!(language_model.supports_explicit_compaction());
assert!(!language_model.supports_explicit_compaction_output_limit());
for includes_status_messages in [false, true] {
let captured_request = Arc::new(Mutex::new(None));
let captured_request_for_handler = captured_request.clone();
let http_client = FakeHttpClient::create(move |request| {
let captured_request = captured_request_for_handler.clone();
async move {
let method = request.method().clone();
let uri = request.uri().to_string();
let authorization = request
.headers()
.get("Authorization")
.and_then(|value| value.to_str().ok())
.map(str::to_string);
let requested_status_messages = request
.headers()
.contains_key(CLIENT_SUPPORTS_STATUS_MESSAGES_HEADER_NAME);
let requested_stream_end = request
.headers()
.contains_key(CLIENT_SUPPORTS_STATUS_STREAM_ENDED_HEADER_NAME);
let mut body = request.into_body();
let mut body_text = String::new();
body.read_to_string(&mut body_text).await?;
*captured_request.lock().unwrap() = Some((
method,
uri,
authorization,
requested_status_messages,
requested_stream_end,
body_text,
));
let result = json!({
"id": "resp_compact",
"created_at": 1_700_000_000,
"object": "response.compaction",
"output": [{
"type": "compaction",
"id": "cmp_manual",
"encrypted_content": "opaque-state"
}],
"usage": {
"input_tokens": 100,
"input_tokens_details": {"cached_tokens": 20},
"output_tokens": 10,
"output_tokens_details": {"reasoning_tokens": 5},
"total_tokens": 110
}
});
let mut response = http_client::Response::builder().status(200);
let body = if includes_status_messages {
response =
response.header(SERVER_SUPPORTS_STATUS_MESSAGES_HEADER_NAME, "true");
[
json!({"status": "keep_alive"}),
json!({"status": "future_transport_status"}),
json!({"event": result}),
json!({"status": "stream_ended"}),
]
.into_iter()
.map(|event| format!("{event}\n"))
.collect::<String>()
} else {
format!("{result}\n")
};
Ok(response.body(AsyncBody::from(body))?)
}
});
let (provider, model) = cloud_test_model(http_client);
let request = compact_test_request();
let result = cx
.update(|cx| provider.compact(&model, request, cx))
.await
.unwrap();
assert_eq!(
result.usage,
language_model::TokenUsage {
input_tokens: 80,
output_tokens: 10,
cache_creation_input_tokens: 0,
cache_read_input_tokens: 20,
}
);
let language_model::CompactedContext::ProviderState(state) = result.context else {
panic!("expected provider compaction state");
};
assert_eq!(
open_ai::responses::provider_compaction_items(&state, &OPEN_AI_PROVIDER_ID)
.unwrap(),
Some(vec![json!({
"type": "compaction",
"id": "cmp_manual",
"encrypted_content": "opaque-state"
})])
);
let (method, uri, authorization, requested_status_messages, requested_stream_end, body) =
captured_request.lock().unwrap().take().unwrap();
assert_eq!(method, Method::POST);
assert_eq!(uri, "http://test.example/completions/compact?");
assert_eq!(authorization.as_deref(), Some("Bearer test-token"));
assert!(requested_status_messages);
assert!(requested_stream_end);
let body = serde_json::from_str::<serde_json::Value>(&body).unwrap();
assert_eq!(body["thread_id"], "thread-123");
assert_eq!(body["provider"], "open_ai");
assert_eq!(body["model"], "gpt-5.4");
assert_eq!(
body["provider_request"],
json!({
"model": "gpt-5.4",
"input": [{
"type": "message",
"role": "user",
"content": [{
"type": "input_text",
"text": "Retain this context."
}]
}],
"prompt_cache_key": "thread-123",
"service_tier": "priority"
})
);
}
}
#[gpui::test]
async fn cloud_explicit_compaction_preserves_streamed_failure(cx: &mut gpui::TestAppContext) {
let http_client = FakeHttpClient::create(|_| async move {
Ok(Response::builder()
.status(200)
.header(SERVER_SUPPORTS_STATUS_MESSAGES_HEADER_NAME, "true")
.body(AsyncBody::from(format!(
"{}\n{}\n",
json!({"status": "keep_alive"}),
json!({"status": {"failed": {
"code": "upstream_http_429",
"message": "Compaction rate limit exceeded.",
"request_id": "00000000-0000-0000-0000-000000000000",
"retry_after": 1.5
}}}),
)))?)
});
let (provider, model) = cloud_test_model(http_client);
let error = cx
.update(|cx| provider.compact(&model, compact_test_request(), cx))
.await
.unwrap_err();
let LanguageModelCompletionError::ProviderRejection {
provider,
status,
code,
message,
retry_after,
category,
} = error
else {
panic!("expected the streamed provider rejection, got {error:?}");
};
assert_eq!(provider, OPEN_AI_PROVIDER_NAME);
assert_eq!(status, Some(StatusCode::TOO_MANY_REQUESTS));
assert_eq!(code.as_deref(), Some("upstream_http_429"));
assert_eq!(message, "Compaction rate limit exceeded.");
assert_eq!(retry_after, Some(Duration::from_secs_f64(1.5)));
assert_eq!(category, ProviderErrorCategory::RateLimit);
}
#[gpui::test]
async fn cloud_explicit_compaction_rejects_stream_ending_without_result(
cx: &mut gpui::TestAppContext,
) {
for stream_ended in [false, true] {
let http_client = FakeHttpClient::create(move |_| async move {
let mut body = format!("{}\n", json!({"status": "keep_alive"}));
if stream_ended {
body.push_str(&format!("{}\n", json!({"status": "stream_ended"})));
}
let body = stream::iter([Ok::<_, std::io::Error>(body.into_bytes())])
.chain(stream::poll_fn(move |_| {
if stream_ended {
Poll::Pending
} else {
Poll::Ready(None)
}
}))
.into_async_read();
Ok(Response::builder()
.status(200)
.header(SERVER_SUPPORTS_STATUS_MESSAGES_HEADER_NAME, "true")
.body(AsyncBody::from_reader(body))?)
});
let (provider, model) = cloud_test_model(http_client);
let error = cx
.update(|cx| provider.compact(&model, compact_test_request(), cx))
.now_or_never()
.expect(
"compaction should stop at EOF or StreamEnded without waiting for more bytes",
)
.unwrap_err();
assert!(
matches!(
&error,
LanguageModelCompletionError::StreamEndedUnexpectedly { provider }
if provider == &OPEN_AI_PROVIDER_NAME
),
"expected a missing compaction result error, got {error:?}"
);
}
}
#[gpui::test]
async fn cloud_anthropic_explicit_compaction_uses_paused_completion(
cx: &mut gpui::TestAppContext,
) {
let captured_request = Arc::new(Mutex::new(None));
let captured_request_for_handler = captured_request.clone();
let http_client = FakeHttpClient::create(move |request| {
let captured_request = captured_request_for_handler.clone();
async move {
let uri = request.uri().to_string();
let mut body = request.into_body();
let mut body_text = String::new();
body.read_to_string(&mut body_text).await?;
*captured_request.lock().unwrap() = Some((uri, body_text));
let response_lines = [
json!({
"event": {
"type": "message_start",
"message": {
"id": "msg_compact",
"type": "message",
"role": "assistant",
"content": [],
"model": "claude-opus-4-6",
"stop_reason": null,
"stop_sequence": null,
"usage": {
"input_tokens": 0,
"output_tokens": 0
}
}
}
}),
json!({
"event": {
"type": "content_block_start",
"index": 0,
"content_block": {
"type": "compaction",
"content": null,
"encrypted_content": null
}
}
}),
json!({
"event": {
"type": "content_block_delta",
"index": 0,
"delta": {
"type": "compaction_delta",
"content": "Summary of the conversation.",
"encrypted_content": "opaque-state"
}
}
}),
json!({
"event": {
"type": "content_block_stop",
"index": 0
}
}),
json!({
"event": {
"type": "message_delta",
"delta": {
"stop_reason": "compaction",
"stop_sequence": null
},
"usage": {
"input_tokens": 0,
"output_tokens": 0,
"iterations": [{
"type": "compaction",
"input_tokens": 60_000,
"output_tokens": 1_000
}]
}
}
}),
json!({"event": {"type": "message_stop"}}),
]
.into_iter()
.map(|line| line["event"].to_string())
.collect::<Vec<_>>()
.join("\n");
Ok(http_client::Response::builder()
.status(200)
.body(AsyncBody::from(format!("{response_lines}\n")))?)
}
});
let (provider, model) = cloud_anthropic_test_model(http_client);
let mut request = compact_test_request();
request.max_output_tokens = Some(8192);
let result = cx
.update(|cx| provider.compact(&model, request, cx))
.await
.unwrap();
assert_eq!(
result.usage,
language_model::TokenUsage {
input_tokens: 60_000,
output_tokens: 1_000,
..Default::default()
}
);
let language_model::CompactedContext::Summary {
content,
provider_state,
} = result.context
else {
panic!("expected summary compaction");
};
assert_eq!(content.as_ref(), "Summary of the conversation.");
assert_eq!(
anthropic::completion::provider_compaction_encrypted_content(
&provider_state.expect("expected opaque provider state"),
&ANTHROPIC_PROVIDER_ID,
)
.unwrap()
.as_deref(),
Some("opaque-state")
);
let (uri, body) = captured_request.lock().unwrap().take().unwrap();
assert_eq!(uri, "http://test.example/completions?");
let body = serde_json::from_str::<serde_json::Value>(&body).unwrap();
assert_eq!(body["provider"], "anthropic");
assert_eq!(body["provider_request"]["max_tokens"], 8192);
assert_eq!(
body["provider_request"]["context_management"],
json!({
"edits": [{
"type": "compact_20260112",
"trigger": {
"type": "input_tokens",
"value": anthropic::MIN_COMPACTION_TRIGGER_TOKENS
},
"pause_after_compaction": true
}]
})
);
assert!(body["provider_request"]["tools"].is_null());
}
#[gpui::test]
async fn cloud_explicit_compaction_rejects_output_without_compaction_item(
cx: &mut gpui::TestAppContext,
) {
let http_client = FakeHttpClient::create(|_| async move {
Ok(http_client::Response::builder()
.status(200)
.body(AsyncBody::from(format!(
"{}\n",
json!({
"id": "resp_compact",
"created_at": 1_700_000_000,
"object": "response.compaction",
"output": [{
"type": "message",
"role": "assistant",
"content": "This is not an opaque compaction item."
}],
"usage": {
"input_tokens": 100,
"input_tokens_details": {"cached_tokens": 20},
"output_tokens": 10,
"output_tokens_details": {"reasoning_tokens": 5},
"total_tokens": 110
}
})
)))?)
});
let (provider, model) = cloud_test_model(http_client);
let error = cx
.update(|cx| provider.compact(&model, compact_test_request(), cx))
.await
.unwrap_err();
assert!(
matches!(&error, LanguageModelCompletionError::Other(_)),
"expected invalid canonical output to be rejected, got {error:?}"
);
assert!(error.to_string().contains("compaction item"));
}
#[test]
fn test_api_error_conversion_with_upstream_http_error() {
// upstream_http_error with 503 status should become an Overloaded rejection
let error_body = r#"{"code":"upstream_http_error","message":"Received an error from the Anthropic API: upstream connect error or disconnect/reset before headers, reset reason: connection timeout","upstream_status":503}"#;
let api_error = ApiError {
status: StatusCode::INTERNAL_SERVER_ERROR,
body: error_body.to_string(),
headers: HeaderMap::new(),
};
let completion_error: LanguageModelCompletionError = api_error.into();
match completion_error {
LanguageModelCompletionError::ProviderRejection {
provider,
retry_after,
category: ProviderErrorCategory::Overloaded,
..
} => {
assert_eq!(provider, PROVIDER_NAME);
assert_eq!(retry_after, None);
}
_ => panic!(
"Expected Overloaded rejection for upstream 503, got: {:?}",
completion_error
),
}
// upstream_http_error with 500 status should preserve the rejection
let error_body = r#"{"code":"upstream_http_error","message":"Received an error from the OpenAI API: internal server error","upstream_status":500}"#;
let api_error = ApiError {
status: StatusCode::INTERNAL_SERVER_ERROR,
body: error_body.to_string(),
headers: HeaderMap::new(),
};
let completion_error: LanguageModelCompletionError = api_error.into();
match completion_error {
LanguageModelCompletionError::ProviderRejection {
provider,
status,
code,
message,
retry_after,
category,
} => {
assert_eq!(provider, PROVIDER_NAME);
assert_eq!(status, Some(StatusCode::INTERNAL_SERVER_ERROR));
assert_eq!(code.as_deref(), Some("upstream_http_error"));
assert_eq!(
message,
"Received an error from the OpenAI API: internal server error"
);
assert_eq!(retry_after, None);
assert_eq!(category, ProviderErrorCategory::InternalServer);
}
_ => panic!(
"Expected ProviderRejection for upstream 500, got: {:?}",
completion_error
),
}
// upstream_http_error with 429 status should become a RateLimit rejection
let error_body = r#"{"code":"upstream_http_error","message":"Received an error from the Google API: rate limit exceeded","upstream_status":429}"#;
let api_error = ApiError {
status: StatusCode::INTERNAL_SERVER_ERROR,
body: error_body.to_string(),
headers: HeaderMap::new(),
};
let completion_error: LanguageModelCompletionError = api_error.into();
match completion_error {
LanguageModelCompletionError::ProviderRejection {
provider,
retry_after,
category: ProviderErrorCategory::RateLimit,
..
} => {
assert_eq!(provider, PROVIDER_NAME);
assert_eq!(retry_after, None);
}
_ => panic!(
"Expected RateLimit rejection for upstream 429, got: {:?}",
completion_error
),
}
// Regular 500 error without upstream_http_error should preserve the Zed rejection
let error_body = "Regular internal server error";
let api_error = ApiError {
status: StatusCode::INTERNAL_SERVER_ERROR,
body: error_body.to_string(),
headers: HeaderMap::new(),
};
let completion_error: LanguageModelCompletionError = api_error.into();
match completion_error {
LanguageModelCompletionError::ProviderRejection {
provider,
status,
code,
message,
retry_after,
category,
} => {
assert_eq!(provider, PROVIDER_NAME);
assert_eq!(status, Some(StatusCode::INTERNAL_SERVER_ERROR));
assert_eq!(code, None);
assert_eq!(message, "Regular internal server error");
assert_eq!(retry_after, None);
assert_eq!(category, ProviderErrorCategory::InternalServer);
}
_ => panic!(
"Expected ProviderRejection for regular 500, got: {:?}",
completion_error
),
}
// upstream_http_429 format should be converted to a RateLimit rejection
let error_body = r#"{"code":"upstream_http_429","message":"Upstream Anthropic rate limit exceeded.","retry_after":30.5}"#;
let api_error = ApiError {
status: StatusCode::INTERNAL_SERVER_ERROR,
body: error_body.to_string(),
headers: HeaderMap::new(),
};
let completion_error: LanguageModelCompletionError = api_error.into();
match completion_error {
LanguageModelCompletionError::ProviderRejection {
provider,
retry_after,
category: ProviderErrorCategory::RateLimit,
..
} => {
assert_eq!(provider, PROVIDER_NAME);
assert_eq!(retry_after, Some(Duration::from_secs_f64(30.5)));
}
_ => panic!(
"Expected RateLimit rejection for upstream_http_429, got: {:?}",
completion_error
),
}
// Invalid JSON in error body should fall back to regular error handling
let error_body = "Not JSON at all";
let api_error = ApiError {
status: StatusCode::INTERNAL_SERVER_ERROR,
body: error_body.to_string(),
headers: HeaderMap::new(),
};
let completion_error: LanguageModelCompletionError = api_error.into();
match completion_error {
LanguageModelCompletionError::ProviderRejection {
provider,
status,
code,
message,
..
} => {
assert_eq!(provider, PROVIDER_NAME);
assert_eq!(status, Some(StatusCode::INTERNAL_SERVER_ERROR));
assert_eq!(code, None);
assert_eq!(message, "Not JSON at all");
}
_ => panic!(
"Expected ProviderRejection for invalid JSON, got: {:?}",
completion_error
),
}
}
#[gpui::test]
async fn cloud_transport_errors_include_hostname(cx: &mut gpui::TestAppContext) {
let http_client =
FakeHttpClient::create(|_| async move { Err(anyhow::anyhow!("DNS lookup failed")) });
let (provider, model) = cloud_test_model(http_client);
let error = cx
.update(|cx| provider.compact(&model, compact_test_request(), cx))
.await
.expect_err("request should fail");
assert!(matches!(
error,
LanguageModelCompletionError::HttpSend {
host,
error,
..
} if host == "test.example" && error.to_string() == "DNS lookup failed"
));
}
#[gpui::test]
async fn token_acquisition_failures_are_classified(_cx: &mut gpui::TestAppContext) {
async fn token_failure(error: fn() -> ClientApiError) -> LanguageModelCompletionError {
let http_client = FakeHttpClient::create(|_| async move {
panic!("no request should be sent without a token")
});
let http_client = HttpClientWithUrl::new(http_client, "https://test.example", None);
authenticated_llm_request(
&http_client,
&FailingTokenProvider { error },
(),
|_token| unreachable!("no request should be built without a token"),
)
.await
.err()
.expect("token acquisition should fail")
.into_completion_error("test.example".to_string())
}
for error in [
(|| ClientApiError::Unauthorized) as fn() -> ClientApiError,
|| ClientApiError::NotSignedIn,
] {
let error = token_failure(error).await;
assert!(
matches!(
&error,
LanguageModelCompletionError::ProviderRejection {
provider,
category: ProviderErrorCategory::Authentication,
..
} if *provider == PROVIDER_NAME
),
"{error:?}"
);
assert!(!error.is_transient());
}
let error = token_failure(|| ClientApiError::ServerError {
host: "test.example".to_string(),
status: StatusCode::PAYMENT_REQUIRED,
body: "no subscription".to_string(),
})
.await;
assert!(
matches!(
error,
LanguageModelCompletionError::ProviderRejection {
category: ProviderErrorCategory::PaymentRequired,
..
}
),
"{error:?}"
);
let error = token_failure(|| ClientApiError::ServerError {
host: "test.example".to_string(),
status: StatusCode::FORBIDDEN,
body: "no model access".to_string(),
})
.await;
assert!(
matches!(
&error,
LanguageModelCompletionError::ProviderRejection {
category: ProviderErrorCategory::Permission,
message,
..
} if message == "no model access"
),
"{error:?}"
);
let error = token_failure(|| ClientApiError::ConnectionFailed {
host: "test.example".to_string(),
source: anyhow::anyhow!("offline"),
})
.await;
assert!(
matches!(
&error,
LanguageModelCompletionError::HttpSend { host, .. } if host == "test.example"
),
"{error:?}"
);
assert!(error.is_transient());
}
#[test]
fn test_response_stream_error_maps_to_structured_variant() {
// Read/deserialize failures mid-stream must keep their structured
// variant rather than collapsing into `Other` (the source of the
// generic "Request failed." message).
let read = ResponseStreamError::Read(std::io::Error::from(std::io::ErrorKind::BrokenPipe))
.into_completion_error(PROVIDER_NAME);
assert!(
matches!(
read,
LanguageModelCompletionError::ApiReadResponseError { .. }
),
"Expected ApiReadResponseError, got: {read:?}"
);
let deserialize = ResponseStreamError::Deserialize(
serde_json::from_str::<serde_json::Value>("not json").unwrap_err(),
)
.into_completion_error(PROVIDER_NAME);
assert!(
matches!(
deserialize,
LanguageModelCompletionError::DeserializeResponse { .. }
),
"Expected DeserializeResponse, got: {deserialize:?}"
);
}
#[test]
fn cloud_anthropic_supports_explicit_compaction_after_minimum_input() {
let model = language_model(&cloud_anthropic_test_config());
assert!(model.supports_explicit_compaction);
assert!(model.supports_explicit_compaction_output_limit);
assert_eq!(
model.minimum_explicit_compaction_input_tokens(),
Some(anthropic::MIN_COMPACTION_TRIGGER_TOKENS)
);
}
#[test]
fn prompt_compaction_preserves_prefix_only_for_supported_upstreams() {
assert_eq!(
language_model(&cloud_test_config()).prompt_compaction_strategy,
PromptCompactionStrategy::PreserveRequestPrefix
);
assert_eq!(
language_model(&cloud_anthropic_test_config()).prompt_compaction_strategy,
PromptCompactionStrategy::RebuildPrompt
);
}
#[test]
fn config_resolves_current_model_list_and_rejects_unlisted_models() {
let listed = language_model(&cloud_test_config());
let mut provider = test_provider(
FakeHttpClient::with_404_response(),
vec![cloud_test_config(), cloud_anthropic_test_config()],
);
assert_eq!(provider.config(&listed).unwrap().id.0.as_ref(), "gpt-5.4");
let mut refreshed = cloud_test_config();
refreshed.max_output_tokens = 64_000;
provider.update_models(ListModelsResponse {
models: vec![refreshed],
default_model: None,
default_fast_model: None,
recommended_models: Vec::new(),
});
assert_eq!(provider.config(&listed).unwrap().max_output_tokens, 64_000);
let unlisted = language_model(&cloud_anthropic_test_config());
assert!(matches!(
provider.config(&unlisted),
Err(LanguageModelCompletionError::ModelUnavailable { provider, model })
if provider == PROVIDER_NAME && model == unlisted.id
));
}
#[gpui::test]
async fn hosted_input_counts_use_authenticated_count_endpoint(cx: &mut gpui::TestAppContext) {
for anthropic in [true, false] {
let calls = Arc::new(std::sync::atomic::AtomicUsize::new(0));
let http_client = FakeHttpClient::create({
let calls = calls.clone();
move |mut request| {
let calls = calls.clone();
async move {
calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
assert_eq!(request.uri().path(), "/count_tokens");
assert_eq!(request.headers()["Authorization"], "Bearer test-token");
let mut body = String::new();
request.body_mut().read_to_string(&mut body).await?;
let body: serde_json::Value = serde_json::from_str(&body)?;
let model = if anthropic {
"claude-opus-4-6"
} else {
"gpt-5.4"
};
assert_eq!(body["model"], model);
assert_eq!(body["provider_request"]["model"], model);
assert!(body["provider_request"].to_string().contains("aW1hZ2U="));
Ok(Response::builder()
.status(200)
.body(AsyncBody::from(r#"{"tokens":731}"#))?)
}
}
});
let (provider, model) = if anthropic {
cloud_anthropic_test_model(http_client)
} else {
cloud_test_model(http_client)
};
let mut request = compact_test_request();
request.messages[0].content.push(MessageContent::Image(
language_model::LanguageModelImage {
source: "aW1hZ2U=".into(),
},
));
assert_eq!(
cx.update(|cx| provider.count_input_tokens(&model, request, cx))
.await
.unwrap(),
Some(731)
);
assert_eq!(calls.load(std::sync::atomic::Ordering::SeqCst), 1);
}
}
#[gpui::test]
async fn hosted_input_counts_surface_errors_and_skip_unsupported_providers(
cx: &mut gpui::TestAppContext,
) {
for (status, body) in [(429, "rate limit"), (200, r#"{"tokens":-1}"#), (200, "{}")] {
let (provider, model) = cloud_test_model(FakeHttpClient::create(move |_| async move {
Ok(Response::builder()
.status(status)
.body(AsyncBody::from(body))?)
}));
let error = cx
.update(|cx| provider.count_input_tokens(&model, compact_test_request(), cx))
.await
.unwrap_err();
if status == 200 {
assert!(matches!(
error,
LanguageModelCompletionError::DeserializeResponse { .. }
));
} else {
assert!(matches!(
error,
LanguageModelCompletionError::ProviderRejection {
category: ProviderErrorCategory::RateLimit,
..
}
));
}
}
let mut config = cloud_test_config();
config.provider = cloud_llm_client::LanguageModelProvider::XAi;
let (provider, model) = bind_test_model(
FakeHttpClient::create(|_| async {
panic!("unsupported provider must not send a counting request");
}),
config,
);
assert_eq!(
cx.update(|cx| provider.count_input_tokens(&model, compact_test_request(), cx))
.await
.unwrap(),
None
);
}
#[test]
fn opus_55_hosted_requests_use_adaptive_thinking_with_prefix_binding() -> Result<()> {
let mut config = cloud_anthropic_test_config();
config.id = cloud_llm_client::LanguageModelId("claude-opus-5-5".into());
for effort in [None, Some("high")] {
let request = anthropic_request(
&config,
LanguageModelRequest {
thinking_allowed: true,
thinking_effort: effort.map(str::to_string),
temperature: Some(0.25),
..Default::default()
},
)?;
let body = serde_json::to_value(request)?;
assert_eq!(body["thinking"]["type"], "adaptive");
assert_eq!(
body["thinking"]["block_binding"]["prefix_mismatch_behavior"],
"drop_block"
);
assert_eq!(body["output_config"]["effort"].as_str(), effort);
assert!(body.get("temperature").is_none());
}
Ok(())
}
#[gpui::test]
async fn retention_consent_gates_counting_compaction_and_generation(
cx: &mut gpui::TestAppContext,
) {
let calls = Arc::new(std::sync::atomic::AtomicUsize::new(0));
let mut config = cloud_anthropic_test_config();
config.id = cloud_llm_client::LanguageModelId("claude-fable-5-1".into());
assert!(language_model(&config).requires_data_retention());
let (provider, model) = bind_test_model(
FakeHttpClient::create({
let calls = calls.clone();
move |request| {
calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
async move {
assert_eq!(request.uri().path(), "/count_tokens");
Ok(Response::builder()
.status(200)
.body(AsyncBody::from(r#"{"tokens":731}"#))?)
}
}
}),
config,
);
let mut request = compact_test_request();
request.messages[0].content.push(MessageContent::Image(
language_model::LanguageModelImage {
source: "aW1hZ2U=".into(),
},
));
assert!(matches!(
cx.update(|cx| provider.count_input_tokens(&model, request.clone(), cx))
.await,
Err(LanguageModelCompletionError::DataRetentionConsentRequired { .. })
));
assert!(matches!(
cx.update(|cx| provider.compact(&model, request.clone(), cx))
.await,
Err(LanguageModelCompletionError::DataRetentionConsentRequired { .. })
));
assert!(matches!(
cx.update(|cx| provider.stream_completion(&model, request.clone(), cx))
.await,
Err(LanguageModelCompletionError::DataRetentionConsentRequired { .. })
));
assert_eq!(calls.load(std::sync::atomic::Ordering::SeqCst), 0);
provider
.token_provider
.data_retention_consent
.store(true, Ordering::SeqCst);
assert_eq!(
cx.update(|cx| provider.count_input_tokens(&model, request, cx))
.await
.unwrap(),
Some(731)
);
assert_eq!(calls.load(std::sync::atomic::Ordering::SeqCst), 1);
}
#[gpui::test]
async fn hosted_open_ai_preserves_unset_output_and_clamps_explicit_caps(
cx: &mut gpui::TestAppContext,
) {
for (limit, expected) in [
(None, None),
(Some(8192), Some(8192)),
(Some(u64::MAX), Some(128_000)),
] {
let (provider, model) =
cloud_test_model(FakeHttpClient::create(move |mut request| async move {
assert_eq!(request.uri().path(), "/completions");
let mut body = String::new();
request.body_mut().read_to_string(&mut body).await?;
let body: serde_json::Value = serde_json::from_str(&body)?;
assert_eq!(
body["provider_request"].get("max_output_tokens").cloned(),
expected.map(|value| json!(value))
);
let completed = CompletionEvent::Event(json!({
"type": "response.completed",
"response": {
"id": "response-1", "status": "completed", "output": [],
"usage": {"input_tokens": 1, "output_tokens": 0, "total_tokens": 1}
}
}));
let ended = CompletionEvent::<serde_json::Value>::Status(
CompletionRequestStatus::StreamEnded,
);
Ok(Response::builder()
.status(200)
.header(SERVER_SUPPORTS_STATUS_MESSAGES_HEADER_NAME, "true")
.body(AsyncBody::from(format!(
"{}\n{}\n",
serde_json::to_string(&completed)?,
serde_json::to_string(&ended)?,
)))?)
}));
let mut request = compact_test_request();
request.max_output_tokens = limit;
let mut stream = cx
.update(|cx| provider.stream_completion(&model, request, cx))
.await
.unwrap();
while let Some(event) = stream.next().await {
event.unwrap();
}
}
}
#[gpui::test]
async fn hosted_google_requests_summaries_using_catalog_thinking_support(
cx: &mut gpui::TestAppContext,
) {
let mut config = cloud_test_config();
config.provider = cloud_llm_client::LanguageModelProvider::Google;
config.id = cloud_llm_client::LanguageModelId("future-google-model".into());
config.supports_thinking = true;
let (provider, model) = bind_test_model(
FakeHttpClient::create(|mut request| async move {
let mut body = String::new();
request.body_mut().read_to_string(&mut body).await?;
let body: serde_json::Value = serde_json::from_str(&body)?;
assert_eq!(
body["provider_request"]["generationConfig"]["thinkingConfig"],
json!({"thinkingLevel": "MEDIUM", "includeThoughts": true})
);
let completed = CompletionEvent::Event(json!({
"candidates": [{
"content": {
"role": "model",
"parts": [{"text": "Summary", "thought": true}, {"text": "Answer"}]
},
"finishReason": "STOP"
}]
}));
let ended = CompletionEvent::<serde_json::Value>::Status(
CompletionRequestStatus::StreamEnded,
);
Ok(Response::builder()
.status(200)
.header(SERVER_SUPPORTS_STATUS_MESSAGES_HEADER_NAME, "true")
.body(AsyncBody::from(format!(
"{}\n{}\n",
serde_json::to_string(&completed)?,
serde_json::to_string(&ended)?,
)))?)
}),
config,
);
let mut request = compact_test_request();
request.thinking_allowed = true;
request.thinking_effort = Some("medium".into());
let mut stream = cx
.update(|cx| provider.stream_completion(&model, request, cx))
.await
.unwrap();
let mut summaries = Vec::new();
while let Some(event) = stream.next().await {
if let LanguageModelCompletionEvent::Thinking { text, .. } = event.unwrap() {
summaries.push(text);
}
}
assert_eq!(summaries, ["Summary"]);
}
fn cloud_google_test_model(
events: Vec<serde_json::Value>,
) -> (CloudModelProvider<TestTokenProvider>, LanguageModel) {
let body = events
.into_iter()
.map(|event| format!("{event}\n"))
.collect::<String>();
let http_client = FakeHttpClient::create(move |_| {
let body = body.clone();
async move {
Ok(Response::builder()
.status(200)
.header(SERVER_SUPPORTS_STATUS_MESSAGES_HEADER_NAME, "true")
.body(AsyncBody::from(body))?)
}
});
let mut config = cloud_test_config();
config.provider = cloud_llm_client::LanguageModelProvider::Google;
config.id = cloud_llm_client::LanguageModelId(Arc::from("gemini-3.1-pro-preview"));
bind_test_model(http_client, config)
}
fn compact_test_request() -> LanguageModelRequest {
LanguageModelRequest {
thread_id: Some("thread-123".to_string()),
messages: vec![LanguageModelRequestMessage {
role: Role::User,
content: vec![MessageContent::Text("Retain this context.".to_string())],
cache: false,
reasoning_details: None,
}],
speed: Some(Speed::Fast),
..Default::default()
}
}
fn cloud_anthropic_test_model(
http_client: Arc<HttpClientWithUrl>,
) -> (CloudModelProvider<TestTokenProvider>, LanguageModel) {
bind_test_model(http_client, cloud_anthropic_test_config())
}
fn cloud_test_model(
http_client: Arc<HttpClientWithUrl>,
) -> (CloudModelProvider<TestTokenProvider>, LanguageModel) {
bind_test_model(http_client, cloud_test_config())
}
/// Lists `configs` in a provider's model list, as a successful `/models`
/// fetch would.
fn test_provider(
http_client: Arc<HttpClientWithUrl>,
configs: Vec<cloud_llm_client::LanguageModel>,
) -> CloudModelProvider<TestTokenProvider> {
let mut provider =
CloudModelProvider::new(Arc::new(TestTokenProvider::default()), http_client, None);
provider.update_models(ListModelsResponse {
models: configs,
default_model: None,
default_fast_model: None,
recommended_models: Vec::new(),
});
provider
}
/// Lists `config` in a provider, and describes the model it lists.
fn bind_test_model(
http_client: Arc<HttpClientWithUrl>,
config: cloud_llm_client::LanguageModel,
) -> (CloudModelProvider<TestTokenProvider>, LanguageModel) {
let model = language_model(&config);
(test_provider(http_client, vec![config]), model)
}
#[test]
fn test_disabling_thinking_support_uses_effort_level_override() {
let mut config = cloud_anthropic_test_config();
config.supports_disabling_thinking = false;
config.supported_effort_levels = vec![
cloud_llm_client::SupportedEffortLevel {
name: Arc::from("High"),
value: Arc::from("high"),
is_default: Some(true),
supports_disabling_thinking: Some(true),
},
cloud_llm_client::SupportedEffortLevel {
name: Arc::from("Max"),
value: Arc::from("max"),
is_default: None,
supports_disabling_thinking: None,
},
];
let model = language_model(&config);
assert!(model.supports_disabling_thinking_at_effort(None));
assert!(model.supports_disabling_thinking_at_effort(Some("high")));
assert!(!model.supports_disabling_thinking_at_effort(Some("max")));
assert!(!model.supports_disabling_thinking_at_effort(Some("unknown")));
}
fn cloud_anthropic_test_config() -> cloud_llm_client::LanguageModel {
cloud_llm_client::LanguageModel {
provider: cloud_llm_client::LanguageModelProvider::Anthropic,
id: cloud_llm_client::LanguageModelId(Arc::from("claude-opus-4-6")),
display_name: "Claude Opus 4.6".to_string(),
is_latest: true,
max_token_count: 1_000_000,
max_token_count_in_max_mode: None,
max_output_tokens: 128_000,
supports_tools: true,
supports_images: true,
supports_thinking: true,
supports_disabling_thinking: true,
supports_fast_mode: true,
supports_server_side_compaction: true,
supported_effort_levels: Vec::new(),
supports_streaming_tools: true,
supports_parallel_tool_calls: false,
is_disabled: false,
disabled_reason: None,
}
}
fn cloud_test_config() -> cloud_llm_client::LanguageModel {
cloud_llm_client::LanguageModel {
provider: cloud_llm_client::LanguageModelProvider::OpenAi,
id: cloud_llm_client::LanguageModelId(Arc::from("gpt-5.4")),
display_name: "GPT-5.4".to_string(),
is_latest: true,
max_token_count: 1_000_000,
max_token_count_in_max_mode: None,
max_output_tokens: 128_000,
supports_tools: true,
supports_images: true,
supports_thinking: true,
supports_disabling_thinking: true,
supports_fast_mode: true,
supports_server_side_compaction: true,
supported_effort_levels: Vec::new(),
supports_streaming_tools: true,
supports_parallel_tool_calls: true,
is_disabled: false,
disabled_reason: None,
}
}
#[derive(Default)]
struct TestTokenProvider {
data_retention_consent: AtomicBool,
}
impl CloudLlmTokenProvider for TestTokenProvider {
type AuthContext = ();
fn auth_context(&self, _cx: &impl AppContext) -> Self::AuthContext {}
fn cached_token(
&self,
_auth_context: Self::AuthContext,
) -> BoxFuture<'static, Result<String, ClientApiError>> {
async { Ok("test-token".to_string()) }.boxed()
}
fn refresh_token(
&self,
_auth_context: Self::AuthContext,
) -> BoxFuture<'static, Result<String, ClientApiError>> {
async { Ok("refreshed-test-token".to_string()) }.boxed()
}
fn has_data_retention_consent(&self, _cx: &impl AppContext) -> bool {
self.data_retention_consent.load(Ordering::SeqCst)
}
}
struct FailingTokenProvider {
error: fn() -> ClientApiError,
}
impl CloudLlmTokenProvider for FailingTokenProvider {
type AuthContext = ();
fn auth_context(&self, _cx: &impl AppContext) -> Self::AuthContext {}
fn cached_token(
&self,
_auth_context: Self::AuthContext,
) -> BoxFuture<'static, Result<String, ClientApiError>> {
let error = (self.error)();
async move { Err(error) }.boxed()
}
fn refresh_token(
&self,
auth_context: Self::AuthContext,
) -> BoxFuture<'static, Result<String, ClientApiError>> {
self.cached_token(auth_context)
}
fn has_data_retention_consent(&self, _cx: &impl AppContext) -> bool {
false
}
}
}