Code at the speed of thought – Zed is a high-performance, multiplayer code editor from the creators of Atom and Tree-sitter.
language_models_cloud: Classify LLM token failures as provider rejections (#64754)
Conrad Irwindd510f99e04e
diff --git a/Cargo.lock b/Cargo.lockindex 0f966f61ef0..3ec313424cd 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -9801,6 +9801,7 @@ version = "0.1.0"dependencies = ["anthropic","anyhow",+ "cloud_api_client","cloud_llm_client","futures 0.3.34","google_ai",diff --git a/crates/client/src/client.rs b/crates/client/src/client.rsindex c12ea19e39c..5fcbfd3f894 100644--- a/crates/client/src/client.rs+++ b/crates/client/src/client.rs@@ -1609,20 +1609,16 @@ impl Client {&self,llm_token: &LlmApiToken,organization_id: OrganizationId,- ) -> Result<String> {+ ) -> Result<String, ClientApiError> {let system_id = self.telemetry().system_id().map(|x| x.to_string());let cloud_client = self.cloud_client();- match llm_token+ let result = llm_token.cached(&cloud_client, system_id, organization_id)- .await- {- Ok(token) => Ok(token),- Err(ClientApiError::Unauthorized) => {- self.request_sign_out();- Err(ClientApiError::Unauthorized).context("Failed to create LLM token")- }- Err(err) => Err(anyhow::Error::from(err)),+ .await;+ if let Err(ClientApiError::Unauthorized) = result {+ self.request_sign_out();}+ result}/// Sends an authenticated request to the Zed LLM service, retrying once@@ -1654,40 +1650,32 @@ impl Client {&self,llm_token: &LlmApiToken,organization_id: OrganizationId,- ) -> Result<String> {+ ) -> Result<String, ClientApiError> {let system_id = self.telemetry().system_id().map(|x| x.to_string());let cloud_client = self.cloud_client();- match llm_token+ let result = llm_token.refresh(&cloud_client, system_id, organization_id)- .await- {- Ok(token) => Ok(token),- Err(ClientApiError::Unauthorized) => {- self.request_sign_out();- return Err(ClientApiError::Unauthorized).context("Failed to create LLM token");- }- Err(err) => return Err(anyhow::Error::from(err)),+ .await;+ if let Err(ClientApiError::Unauthorized) = result {+ self.request_sign_out();}+ result}pub async fn clear_and_refresh_llm_token(&self,llm_token: &LlmApiToken,organization_id: OrganizationId,- ) -> Result<String> {+ ) -> Result<String, ClientApiError> {let system_id = self.telemetry().system_id().map(|x| x.to_string());let cloud_client = self.cloud_client();- match llm_token+ let result = llm_token.clear_and_refresh(&cloud_client, system_id, organization_id)- .await- {- Ok(token) => Ok(token),- Err(ClientApiError::Unauthorized) => {- self.request_sign_out();- return Err(ClientApiError::Unauthorized).context("Failed to create LLM token");- }- Err(err) => return Err(anyhow::Error::from(err)),+ .await;+ if let Err(ClientApiError::Unauthorized) = result {+ self.request_sign_out();}+ result}pub async fn sign_out(self: &Arc<Self>, cx: &AsyncApp) {diff --git a/crates/language_models/src/provider/cloud.rs b/crates/language_models/src/provider/cloud.rsindex 7557ba65de5..7332a2a01f1 100644--- a/crates/language_models/src/provider/cloud.rs+++ b/crates/language_models/src/provider/cloud.rs@@ -3,7 +3,7 @@ use anyhow::{Result, anyhow};use client::{Client, RefreshLlmTokenListener, TelemetrySettings, UserStore, global_llm_token, zed_urls,};-use cloud_api_client::LlmApiToken;+use cloud_api_client::{ClientApiError, LlmApiToken};use cloud_api_types::OrganizationId;use cloud_api_types::Plan;use futures::FutureExt;@@ -53,12 +53,11 @@ impl CloudLlmTokenProvider for ClientTokenProvider {fn cached_token(&self,organization_id: Self::AuthContext,- ) -> BoxFuture<'static, Result<String>> {+ ) -> BoxFuture<'static, Result<String, ClientApiError>> {let client = self.client.clone();let llm_api_token = self.llm_api_token.clone();Box::pin(async move {- let organization_id =- organization_id.ok_or_else(|| anyhow!("No organization selected."))?;+ let organization_id = organization_id.ok_or(ClientApiError::NotSignedIn)?;client.cached_llm_token(&llm_api_token, organization_id).await@@ -68,12 +67,11 @@ impl CloudLlmTokenProvider for ClientTokenProvider {fn refresh_token(&self,organization_id: Self::AuthContext,- ) -> BoxFuture<'static, Result<String>> {+ ) -> BoxFuture<'static, Result<String, ClientApiError>> {let client = self.client.clone();let llm_api_token = self.llm_api_token.clone();Box::pin(async move {- let organization_id =- organization_id.ok_or_else(|| anyhow!("No organization selected."))?;+ let organization_id = organization_id.ok_or(ClientApiError::NotSignedIn)?;client.refresh_llm_token(&llm_api_token, organization_id).awaitdiff --git a/crates/language_models_cloud/Cargo.toml b/crates/language_models_cloud/Cargo.tomlindex e73e79a9a09..250570759b7 100644--- a/crates/language_models_cloud/Cargo.toml+++ b/crates/language_models_cloud/Cargo.toml@@ -15,6 +15,7 @@ doctest = false[dependencies]anthropic = { workspace = true, features = ["schemars"] }anyhow.workspace = true+cloud_api_client.workspace = truecloud_llm_client.workspace = truefutures.workspace = truegoogle_ai = { workspace = true, features = ["schemars"] }diff --git a/crates/language_models_cloud/src/language_models_cloud.rs b/crates/language_models_cloud/src/language_models_cloud.rsindex 0a0dc1ead51..9202c76ceaf 100644--- a/crates/language_models_cloud/src/language_models_cloud.rs+++ b/crates/language_models_cloud/src/language_models_cloud.rs@@ -1,5 +1,6 @@use anthropic::AnthropicModelMode;use anyhow::{Context as _, Result};+use cloud_api_client::ClientApiError;use cloud_llm_client::{CLIENT_SUPPORTS_STATUS_MESSAGES_HEADER_NAME, CLIENT_SUPPORTS_STATUS_STREAM_ENDED_HEADER_NAME,CLIENT_SUPPORTS_X_AI_HEADER_NAME, CompletionBody, CompletionEvent, CompletionRequestStatus,@@ -55,8 +56,14 @@ pub trait CloudLlmTokenProvider: Send + Sync {type AuthContext: Clone + Send + 'static;fn auth_context(&self, cx: &impl AppContext) -> Self::AuthContext;- fn cached_token(&self, auth_context: Self::AuthContext) -> BoxFuture<'static, Result<String>>;- fn refresh_token(&self, auth_context: Self::AuthContext) -> BoxFuture<'static, Result<String>>;+ fn cached_token(+ &self,+ auth_context: Self::AuthContext,+ ) -> BoxFuture<'static, Result<String, ClientApiError>>;+ fn refresh_token(+ &self,+ auth_context: Self::AuthContext,+ ) -> BoxFuture<'static, Result<String, ClientApiError>>;/// Whether the user has consented to upstream providers retaining/// inference logs for models that require it (see@@ -64,6 +71,85 @@ pub trait CloudLlmTokenProvider: Send + Sync {fn has_data_retention_consent(&self, cx: &impl AppContext) -> bool;}+/// Why an authenticated request to the Zed LLM service failed before a+/// response arrived.+#[derive(Debug, Error)]+pub enum LlmRequestError {+ /// The token provider could not supply an LLM token.+ #[error("failed to acquire an LLM token")]+ Token(#[source] ClientApiError),+ #[error("failed to build the LLM request")]+ BuildRequest(#[source] anyhow::Error),+ #[error("failed to send the LLM request")]+ Send(#[source] anyhow::Error),+}++impl LlmRequestError {+ /// Classifies the failure as a completion error for a request to `host`.+ ///+ /// Credential and billing failures while acquiring a token become provider+ /// rejections, so callers can tell them apart from connectivity problems.+ pub fn into_completion_error(self, host: String) -> LanguageModelCompletionError {+ match self {+ Self::Token(ClientApiError::Unauthorized) => {+ LanguageModelCompletionError::from_provider_response(+ PROVIDER_NAME,+ Some(StatusCode::UNAUTHORIZED),+ None,+ "Zed rejected the credentials for this request.".to_string(),+ None,+ ProviderErrorCategory::Authentication,+ )+ }+ Self::Token(ClientApiError::NotSignedIn) => {+ LanguageModelCompletionError::from_provider_response(+ PROVIDER_NAME,+ None,+ None,+ "Not signed in to Zed.".to_string(),+ None,+ ProviderErrorCategory::Authentication,+ )+ }+ Self::Token(ClientApiError::ServerError { status, .. })+ if status == StatusCode::PAYMENT_REQUIRED =>+ {+ payment_required_error(status)+ }+ Self::Token(ClientApiError::ServerError { status, body, .. }) => {+ LanguageModelCompletionError::from_http_status(PROVIDER_NAME, status, body, None)+ }+ Self::Token(error @ ClientApiError::ConnectionFailed { .. }) => {+ LanguageModelCompletionError::HttpSend {+ provider: PROVIDER_NAME,+ host,+ error: error.into(),+ }+ }+ Self::Send(error) => LanguageModelCompletionError::HttpSend {+ provider: PROVIDER_NAME,+ host,+ error,+ },+ error @ (Self::Token(+ ClientApiError::InvalidResponse(_) | ClientApiError::RequestBuildFailed(_),+ )+ | Self::BuildRequest(_)) => LanguageModelCompletionError::Other(error.into()),+ }+ }+}++fn payment_required_error(status: StatusCode) -> LanguageModelCompletionError {+ LanguageModelCompletionError::from_provider_response(+ PROVIDER_NAME,+ Some(status),+ None,+ "payment required to use this language model; please upgrade your account".to_string(),+ None,+ ProviderErrorCategory::PaymentRequired,+ )+}+/// Sends an authenticated request to the Zed LLM service, retrying once with/// a refreshed token if the server signals that the cached LLM token is/// expired or otherwise rejected. Returns the raw response so callers can@@ -73,15 +159,28 @@ pub async fn authenticated_llm_request<TP: CloudLlmTokenProvider>(token_provider: &TP,auth_context: TP::AuthContext,build_request: impl Fn(&str) -> Result<http_client::Request<AsyncBody>>,-) -> Result<Response<AsyncBody>> {- let token = token_provider.cached_token(auth_context.clone()).await?;- let response = http_client.send(build_request(&token)?).await?;+) -> Result<Response<AsyncBody>, LlmRequestError> {+ let send = async |token: String| {+ let request = build_request(&token).map_err(LlmRequestError::BuildRequest)?;+ http_client+ .send(request)+ .await+ .map_err(LlmRequestError::Send)+ };+ let token = token_provider+ .cached_token(auth_context.clone())+ .await+ .map_err(LlmRequestError::Token)?;+ let response = send(token).await?;if !needs_llm_token_refresh(&response) && response.status() != StatusCode::UNAUTHORIZED {return Ok(response);}log::info!("LLM token rejected; refreshing and retrying request");- let token = token_provider.refresh_token(auth_context).await?;- http_client.send(build_request(&token)?).await+ let token = token_provider+ .refresh_token(auth_context)+ .await+ .map_err(LlmRequestError::Token)?;+ send(token).await}#[derive(Default, Clone, Debug, PartialEq, Serialize, Deserialize, JsonSchema)]@@ -185,11 +284,7 @@ impl<TP: CloudLlmTokenProvider> CloudModelProvider<TP> {Ok(request.body(body.clone().into())?)}).await- .map_err(|error| LanguageModelCompletionError::HttpSend {- provider: PROVIDER_NAME,- host,- error,- })?;+ .map_err(|error| error.into_completion_error(host))?;let status = response.status();if status.is_success() {@@ -206,15 +301,7 @@ impl<TP: CloudLlmTokenProvider> CloudModelProvider<TP> {}if status == StatusCode::PAYMENT_REQUIRED {- return Err(LanguageModelCompletionError::from_provider_response(- PROVIDER_NAME,- Some(status),- None,- "payment required to use this language model; please upgrade your account"- .to_string(),- None,- ProviderErrorCategory::PaymentRequired,- ));+ return Err(payment_required_error(status));}let mut body = String::new();@@ -1918,6 +2005,94 @@ mod tests {));}+ #[gpui::test]+ async fn token_acquisition_failures_are_classified(_cx: &mut gpui::TestAppContext) {+ async fn token_failure(error: fn() -> ClientApiError) -> LanguageModelCompletionError {+ let http_client = FakeHttpClient::create(|_| async move {+ panic!("no request should be sent without a token")+ });+ let http_client = HttpClientWithUrl::new(http_client, "https://test.example", None);+ authenticated_llm_request(+ &http_client,+ &FailingTokenProvider { error },+ (),+ |_token| unreachable!("no request should be built without a token"),+ )+ .await+ .err()+ .expect("token acquisition should fail")+ .into_completion_error("test.example".to_string())+ }++ for error in [+ (|| ClientApiError::Unauthorized) as fn() -> ClientApiError,+ || ClientApiError::NotSignedIn,+ ] {+ let error = token_failure(error).await;+ assert!(+ matches!(+ &error,+ LanguageModelCompletionError::ProviderRejection {+ provider,+ category: ProviderErrorCategory::Authentication,+ ..+ } if *provider == PROVIDER_NAME+ ),+ "{error:?}"+ );+ assert!(!error.is_transient());+ }++ let error = token_failure(|| ClientApiError::ServerError {+ host: "test.example".to_string(),+ status: StatusCode::PAYMENT_REQUIRED,+ body: "no subscription".to_string(),+ })+ .await;+ assert!(+ matches!(+ error,+ LanguageModelCompletionError::ProviderRejection {+ category: ProviderErrorCategory::PaymentRequired,+ ..+ }+ ),+ "{error:?}"+ );++ let error = token_failure(|| ClientApiError::ServerError {+ host: "test.example".to_string(),+ status: StatusCode::FORBIDDEN,+ body: "no model access".to_string(),+ })+ .await;+ assert!(+ matches!(+ &error,+ LanguageModelCompletionError::ProviderRejection {+ category: ProviderErrorCategory::Permission,+ message,+ ..+ } if message == "no model access"+ ),+ "{error:?}"+ );++ let error = token_failure(|| ClientApiError::ConnectionFailed {+ host: "test.example".to_string(),+ source: anyhow::anyhow!("offline"),+ })+ .await;+ assert!(+ matches!(+ &error,+ LanguageModelCompletionError::HttpSend { host, .. } if host == "test.example"+ ),+ "{error:?}"+ );+ assert!(error.is_transient());+ }+#[test]fn test_response_stream_error_maps_to_structured_variant() {// Read/deserialize failures mid-stream must keep their structured@@ -2348,14 +2523,14 @@ mod tests {fn cached_token(&self,_auth_context: Self::AuthContext,- ) -> BoxFuture<'static, Result<String>> {+ ) -> BoxFuture<'static, Result<String, ClientApiError>> {async { Ok("test-token".to_string()) }.boxed()}fn refresh_token(&self,_auth_context: Self::AuthContext,- ) -> BoxFuture<'static, Result<String>> {+ ) -> BoxFuture<'static, Result<String, ClientApiError>> {async { Ok("refreshed-test-token".to_string()) }.boxed()}@@ -2363,4 +2538,33 @@ mod tests {self.data_retention_consent.load(Ordering::SeqCst)}}++ struct FailingTokenProvider {+ error: fn() -> ClientApiError,+ }++ impl CloudLlmTokenProvider for FailingTokenProvider {+ type AuthContext = ();++ fn auth_context(&self, _cx: &impl AppContext) -> Self::AuthContext {}++ fn cached_token(+ &self,+ _auth_context: Self::AuthContext,+ ) -> BoxFuture<'static, Result<String, ClientApiError>> {+ let error = (self.error)();+ async move { Err(error) }.boxed()+ }++ fn refresh_token(+ &self,+ auth_context: Self::AuthContext,+ ) -> BoxFuture<'static, Result<String, ClientApiError>> {+ self.cached_token(auth_context)+ }++ fn has_data_retention_consent(&self, _cx: &impl AppContext) -> bool {+ false+ }+ }}