Skip to content

armgit / zed

Public Imported from GitHub
0

Code at the speed of thought – Zed is a high-performance, multiplayer code editor from the creators of Atom and Tree-sitter.

language_models_cloud: Classify LLM token failures as provider rejections (#64754)

Conrad Irwindd510f99e04e
diff --git a/Cargo.lock b/Cargo.lock
index 0f966f61ef0..3ec313424cd 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -9801,6 +9801,7 @@ version = "0.1.0"
dependencies = [
"anthropic",
"anyhow",
+ "cloud_api_client",
"cloud_llm_client",
"futures 0.3.34",
"google_ai",
diff --git a/crates/client/src/client.rs b/crates/client/src/client.rs
index c12ea19e39c..5fcbfd3f894 100644
--- a/crates/client/src/client.rs
+++ b/crates/client/src/client.rs
@@ -1609,20 +1609,16 @@ impl Client {
&self,
llm_token: &LlmApiToken,
organization_id: OrganizationId,
- ) -> Result<String> {
+ ) -> Result<String, ClientApiError> {
let system_id = self.telemetry().system_id().map(|x| x.to_string());
let cloud_client = self.cloud_client();
- match llm_token
+ let result = llm_token
.cached(&cloud_client, system_id, organization_id)
- .await
- {
- Ok(token) => Ok(token),
- Err(ClientApiError::Unauthorized) => {
- self.request_sign_out();
- Err(ClientApiError::Unauthorized).context("Failed to create LLM token")
- }
- Err(err) => Err(anyhow::Error::from(err)),
+ .await;
+ if let Err(ClientApiError::Unauthorized) = result {
+ self.request_sign_out();
}
+ result
}
/// Sends an authenticated request to the Zed LLM service, retrying once
@@ -1654,40 +1650,32 @@ impl Client {
&self,
llm_token: &LlmApiToken,
organization_id: OrganizationId,
- ) -> Result<String> {
+ ) -> Result<String, ClientApiError> {
let system_id = self.telemetry().system_id().map(|x| x.to_string());
let cloud_client = self.cloud_client();
- match llm_token
+ let result = llm_token
.refresh(&cloud_client, system_id, organization_id)
- .await
- {
- Ok(token) => Ok(token),
- Err(ClientApiError::Unauthorized) => {
- self.request_sign_out();
- return Err(ClientApiError::Unauthorized).context("Failed to create LLM token");
- }
- Err(err) => return Err(anyhow::Error::from(err)),
+ .await;
+ if let Err(ClientApiError::Unauthorized) = result {
+ self.request_sign_out();
}
+ result
}
pub async fn clear_and_refresh_llm_token(
&self,
llm_token: &LlmApiToken,
organization_id: OrganizationId,
- ) -> Result<String> {
+ ) -> Result<String, ClientApiError> {
let system_id = self.telemetry().system_id().map(|x| x.to_string());
let cloud_client = self.cloud_client();
- match llm_token
+ let result = llm_token
.clear_and_refresh(&cloud_client, system_id, organization_id)
- .await
- {
- Ok(token) => Ok(token),
- Err(ClientApiError::Unauthorized) => {
- self.request_sign_out();
- return Err(ClientApiError::Unauthorized).context("Failed to create LLM token");
- }
- Err(err) => return Err(anyhow::Error::from(err)),
+ .await;
+ if let Err(ClientApiError::Unauthorized) = result {
+ self.request_sign_out();
}
+ result
}
pub async fn sign_out(self: &Arc<Self>, cx: &AsyncApp) {
diff --git a/crates/language_models/src/provider/cloud.rs b/crates/language_models/src/provider/cloud.rs
index 7557ba65de5..7332a2a01f1 100644
--- a/crates/language_models/src/provider/cloud.rs
+++ b/crates/language_models/src/provider/cloud.rs
@@ -3,7 +3,7 @@ use anyhow::{Result, anyhow};
use client::{
Client, RefreshLlmTokenListener, TelemetrySettings, UserStore, global_llm_token, zed_urls,
};
-use cloud_api_client::LlmApiToken;
+use cloud_api_client::{ClientApiError, LlmApiToken};
use cloud_api_types::OrganizationId;
use cloud_api_types::Plan;
use futures::FutureExt;
@@ -53,12 +53,11 @@ impl CloudLlmTokenProvider for ClientTokenProvider {
fn cached_token(
&self,
organization_id: Self::AuthContext,
- ) -> BoxFuture<'static, Result<String>> {
+ ) -> BoxFuture<'static, Result<String, ClientApiError>> {
let client = self.client.clone();
let llm_api_token = self.llm_api_token.clone();
Box::pin(async move {
- let organization_id =
- organization_id.ok_or_else(|| anyhow!("No organization selected."))?;
+ let organization_id = organization_id.ok_or(ClientApiError::NotSignedIn)?;
client
.cached_llm_token(&llm_api_token, organization_id)
.await
@@ -68,12 +67,11 @@ impl CloudLlmTokenProvider for ClientTokenProvider {
fn refresh_token(
&self,
organization_id: Self::AuthContext,
- ) -> BoxFuture<'static, Result<String>> {
+ ) -> BoxFuture<'static, Result<String, ClientApiError>> {
let client = self.client.clone();
let llm_api_token = self.llm_api_token.clone();
Box::pin(async move {
- let organization_id =
- organization_id.ok_or_else(|| anyhow!("No organization selected."))?;
+ let organization_id = organization_id.ok_or(ClientApiError::NotSignedIn)?;
client
.refresh_llm_token(&llm_api_token, organization_id)
.await
diff --git a/crates/language_models_cloud/Cargo.toml b/crates/language_models_cloud/Cargo.toml
index e73e79a9a09..250570759b7 100644
--- a/crates/language_models_cloud/Cargo.toml
+++ b/crates/language_models_cloud/Cargo.toml
@@ -15,6 +15,7 @@ doctest = false
[dependencies]
anthropic = { workspace = true, features = ["schemars"] }
anyhow.workspace = true
+cloud_api_client.workspace = true
cloud_llm_client.workspace = true
futures.workspace = true
google_ai = { workspace = true, features = ["schemars"] }
diff --git a/crates/language_models_cloud/src/language_models_cloud.rs b/crates/language_models_cloud/src/language_models_cloud.rs
index 0a0dc1ead51..9202c76ceaf 100644
--- a/crates/language_models_cloud/src/language_models_cloud.rs
+++ b/crates/language_models_cloud/src/language_models_cloud.rs
@@ -1,5 +1,6 @@
use anthropic::AnthropicModelMode;
use anyhow::{Context as _, Result};
+use cloud_api_client::ClientApiError;
use cloud_llm_client::{
CLIENT_SUPPORTS_STATUS_MESSAGES_HEADER_NAME, CLIENT_SUPPORTS_STATUS_STREAM_ENDED_HEADER_NAME,
CLIENT_SUPPORTS_X_AI_HEADER_NAME, CompletionBody, CompletionEvent, CompletionRequestStatus,
@@ -55,8 +56,14 @@ pub trait CloudLlmTokenProvider: Send + Sync {
type AuthContext: Clone + Send + 'static;
fn auth_context(&self, cx: &impl AppContext) -> Self::AuthContext;
- fn cached_token(&self, auth_context: Self::AuthContext) -> BoxFuture<'static, Result<String>>;
- fn refresh_token(&self, auth_context: Self::AuthContext) -> BoxFuture<'static, Result<String>>;
+ fn cached_token(
+ &self,
+ auth_context: Self::AuthContext,
+ ) -> BoxFuture<'static, Result<String, ClientApiError>>;
+ fn refresh_token(
+ &self,
+ auth_context: Self::AuthContext,
+ ) -> BoxFuture<'static, Result<String, ClientApiError>>;
/// Whether the user has consented to upstream providers retaining
/// inference logs for models that require it (see
@@ -64,6 +71,85 @@ pub trait CloudLlmTokenProvider: Send + Sync {
fn has_data_retention_consent(&self, cx: &impl AppContext) -> bool;
}
+/// Why an authenticated request to the Zed LLM service failed before a
+/// response arrived.
+#[derive(Debug, Error)]
+pub enum LlmRequestError {
+ /// The token provider could not supply an LLM token.
+ #[error("failed to acquire an LLM token")]
+ Token(#[source] ClientApiError),
+ #[error("failed to build the LLM request")]
+ BuildRequest(#[source] anyhow::Error),
+ #[error("failed to send the LLM request")]
+ Send(#[source] anyhow::Error),
+}
+
+impl LlmRequestError {
+ /// Classifies the failure as a completion error for a request to `host`.
+ ///
+ /// Credential and billing failures while acquiring a token become provider
+ /// rejections, so callers can tell them apart from connectivity problems.
+ pub fn into_completion_error(self, host: String) -> LanguageModelCompletionError {
+ match self {
+ Self::Token(ClientApiError::Unauthorized) => {
+ LanguageModelCompletionError::from_provider_response(
+ PROVIDER_NAME,
+ Some(StatusCode::UNAUTHORIZED),
+ None,
+ "Zed rejected the credentials for this request.".to_string(),
+ None,
+ ProviderErrorCategory::Authentication,
+ )
+ }
+ Self::Token(ClientApiError::NotSignedIn) => {
+ LanguageModelCompletionError::from_provider_response(
+ PROVIDER_NAME,
+ None,
+ None,
+ "Not signed in to Zed.".to_string(),
+ None,
+ ProviderErrorCategory::Authentication,
+ )
+ }
+ Self::Token(ClientApiError::ServerError { status, .. })
+ if status == StatusCode::PAYMENT_REQUIRED =>
+ {
+ payment_required_error(status)
+ }
+ Self::Token(ClientApiError::ServerError { status, body, .. }) => {
+ LanguageModelCompletionError::from_http_status(PROVIDER_NAME, status, body, None)
+ }
+ Self::Token(error @ ClientApiError::ConnectionFailed { .. }) => {
+ LanguageModelCompletionError::HttpSend {
+ provider: PROVIDER_NAME,
+ host,
+ error: error.into(),
+ }
+ }
+ Self::Send(error) => LanguageModelCompletionError::HttpSend {
+ provider: PROVIDER_NAME,
+ host,
+ error,
+ },
+ error @ (Self::Token(
+ ClientApiError::InvalidResponse(_) | ClientApiError::RequestBuildFailed(_),
+ )
+ | Self::BuildRequest(_)) => LanguageModelCompletionError::Other(error.into()),
+ }
+ }
+}
+
+fn payment_required_error(status: StatusCode) -> LanguageModelCompletionError {
+ LanguageModelCompletionError::from_provider_response(
+ PROVIDER_NAME,
+ Some(status),
+ None,
+ "payment required to use this language model; please upgrade your account".to_string(),
+ None,
+ ProviderErrorCategory::PaymentRequired,
+ )
+}
+
/// Sends an authenticated request to the Zed LLM service, retrying once with
/// a refreshed token if the server signals that the cached LLM token is
/// expired or otherwise rejected. Returns the raw response so callers can
@@ -73,15 +159,28 @@ pub async fn authenticated_llm_request<TP: CloudLlmTokenProvider>(
token_provider: &TP,
auth_context: TP::AuthContext,
build_request: impl Fn(&str) -> Result<http_client::Request<AsyncBody>>,
-) -> Result<Response<AsyncBody>> {
- let token = token_provider.cached_token(auth_context.clone()).await?;
- let response = http_client.send(build_request(&token)?).await?;
+) -> Result<Response<AsyncBody>, LlmRequestError> {
+ let send = async |token: String| {
+ let request = build_request(&token).map_err(LlmRequestError::BuildRequest)?;
+ http_client
+ .send(request)
+ .await
+ .map_err(LlmRequestError::Send)
+ };
+ let token = token_provider
+ .cached_token(auth_context.clone())
+ .await
+ .map_err(LlmRequestError::Token)?;
+ let response = send(token).await?;
if !needs_llm_token_refresh(&response) && response.status() != StatusCode::UNAUTHORIZED {
return Ok(response);
}
log::info!("LLM token rejected; refreshing and retrying request");
- let token = token_provider.refresh_token(auth_context).await?;
- http_client.send(build_request(&token)?).await
+ let token = token_provider
+ .refresh_token(auth_context)
+ .await
+ .map_err(LlmRequestError::Token)?;
+ send(token).await
}
#[derive(Default, Clone, Debug, PartialEq, Serialize, Deserialize, JsonSchema)]
@@ -185,11 +284,7 @@ impl<TP: CloudLlmTokenProvider> CloudModelProvider<TP> {
Ok(request.body(body.clone().into())?)
})
.await
- .map_err(|error| LanguageModelCompletionError::HttpSend {
- provider: PROVIDER_NAME,
- host,
- error,
- })?;
+ .map_err(|error| error.into_completion_error(host))?;
let status = response.status();
if status.is_success() {
@@ -206,15 +301,7 @@ impl<TP: CloudLlmTokenProvider> CloudModelProvider<TP> {
}
if status == StatusCode::PAYMENT_REQUIRED {
- return Err(LanguageModelCompletionError::from_provider_response(
- PROVIDER_NAME,
- Some(status),
- None,
- "payment required to use this language model; please upgrade your account"
- .to_string(),
- None,
- ProviderErrorCategory::PaymentRequired,
- ));
+ return Err(payment_required_error(status));
}
let mut body = String::new();
@@ -1918,6 +2005,94 @@ mod tests {
));
}
+ #[gpui::test]
+ async fn token_acquisition_failures_are_classified(_cx: &mut gpui::TestAppContext) {
+ async fn token_failure(error: fn() -> ClientApiError) -> LanguageModelCompletionError {
+ let http_client = FakeHttpClient::create(|_| async move {
+ panic!("no request should be sent without a token")
+ });
+ let http_client = HttpClientWithUrl::new(http_client, "https://test.example", None);
+ authenticated_llm_request(
+ &http_client,
+ &FailingTokenProvider { error },
+ (),
+ |_token| unreachable!("no request should be built without a token"),
+ )
+ .await
+ .err()
+ .expect("token acquisition should fail")
+ .into_completion_error("test.example".to_string())
+ }
+
+ for error in [
+ (|| ClientApiError::Unauthorized) as fn() -> ClientApiError,
+ || ClientApiError::NotSignedIn,
+ ] {
+ let error = token_failure(error).await;
+ assert!(
+ matches!(
+ &error,
+ LanguageModelCompletionError::ProviderRejection {
+ provider,
+ category: ProviderErrorCategory::Authentication,
+ ..
+ } if *provider == PROVIDER_NAME
+ ),
+ "{error:?}"
+ );
+ assert!(!error.is_transient());
+ }
+
+ let error = token_failure(|| ClientApiError::ServerError {
+ host: "test.example".to_string(),
+ status: StatusCode::PAYMENT_REQUIRED,
+ body: "no subscription".to_string(),
+ })
+ .await;
+ assert!(
+ matches!(
+ error,
+ LanguageModelCompletionError::ProviderRejection {
+ category: ProviderErrorCategory::PaymentRequired,
+ ..
+ }
+ ),
+ "{error:?}"
+ );
+
+ let error = token_failure(|| ClientApiError::ServerError {
+ host: "test.example".to_string(),
+ status: StatusCode::FORBIDDEN,
+ body: "no model access".to_string(),
+ })
+ .await;
+ assert!(
+ matches!(
+ &error,
+ LanguageModelCompletionError::ProviderRejection {
+ category: ProviderErrorCategory::Permission,
+ message,
+ ..
+ } if message == "no model access"
+ ),
+ "{error:?}"
+ );
+
+ let error = token_failure(|| ClientApiError::ConnectionFailed {
+ host: "test.example".to_string(),
+ source: anyhow::anyhow!("offline"),
+ })
+ .await;
+ assert!(
+ matches!(
+ &error,
+ LanguageModelCompletionError::HttpSend { host, .. } if host == "test.example"
+ ),
+ "{error:?}"
+ );
+ assert!(error.is_transient());
+ }
+
#[test]
fn test_response_stream_error_maps_to_structured_variant() {
// Read/deserialize failures mid-stream must keep their structured
@@ -2348,14 +2523,14 @@ mod tests {
fn cached_token(
&self,
_auth_context: Self::AuthContext,
- ) -> BoxFuture<'static, Result<String>> {
+ ) -> BoxFuture<'static, Result<String, ClientApiError>> {
async { Ok("test-token".to_string()) }.boxed()
}
fn refresh_token(
&self,
_auth_context: Self::AuthContext,
- ) -> BoxFuture<'static, Result<String>> {
+ ) -> BoxFuture<'static, Result<String, ClientApiError>> {
async { Ok("refreshed-test-token".to_string()) }.boxed()
}
@@ -2363,4 +2538,33 @@ mod tests {
self.data_retention_consent.load(Ordering::SeqCst)
}
}
+
+ struct FailingTokenProvider {
+ error: fn() -> ClientApiError,
+ }
+
+ impl CloudLlmTokenProvider for FailingTokenProvider {
+ type AuthContext = ();
+
+ fn auth_context(&self, _cx: &impl AppContext) -> Self::AuthContext {}
+
+ fn cached_token(
+ &self,
+ _auth_context: Self::AuthContext,
+ ) -> BoxFuture<'static, Result<String, ClientApiError>> {
+ let error = (self.error)();
+ async move { Err(error) }.boxed()
+ }
+
+ fn refresh_token(
+ &self,
+ auth_context: Self::AuthContext,
+ ) -> BoxFuture<'static, Result<String, ClientApiError>> {
+ self.cached_token(auth_context)
+ }
+
+ fn has_data_retention_consent(&self, _cx: &impl AppContext) -> bool {
+ false
+ }
+ }
}