Code at the speed of thought – Zed is a high-performance, multiplayer code editor from the creators of Atom and Tree-sitter.
crates_io_client: Add crates.io API client (#66437)
Finn Eversccb25cbc6376
diff --git a/Cargo.lock b/Cargo.lockindex a069b149791..ed76ba16512 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -4098,6 +4098,23 @@ dependencies = ["zstd",]+[[package]]+name = "crates_io_client"+version = "0.1.0"+dependencies = [+ "anyhow",+ "async-trait",+ "futures 0.3.34",+ "http_client",+ "itertools 0.14.0",+ "parking_lot",+ "serde",+ "serde_json",+ "smol",+ "thiserror 2.0.17",+ "url",+]+[[package]]name = "crc"version = "3.3.0"diff --git a/Cargo.toml b/Cargo.tomlindex 5b1f5473e6d..b612bdf8744 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -268,6 +268,7 @@ members = [#"tooling/compliance",+ "tooling/crates_io_client","tooling/perf","tooling/xtask",]diff --git a/tooling/crates_io_client/Cargo.toml b/tooling/crates_io_client/Cargo.tomlnew file mode 100644index 00000000000..3928072748f--- /dev/null+++ b/tooling/crates_io_client/Cargo.toml@@ -0,0 +1,33 @@+[package]+name = "crates_io_client"+version = "0.1.0"+edition.workspace = true+publish.workspace = true+license = "Apache-2.0"+description = "crates.io <https://crates.io> API client"++[lints]+workspace = true++[features]+test-support = []++[lib]+path = "src/crates_io_client.rs"+doctest = false++[dependencies]+anyhow.workspace = true+async-trait.workspace = true+futures.workspace = true+http_client.workspace = true+itertools.workspace = true+parking_lot.workspace = true+serde = { workspace = true, features = ["derive"] }+serde_json.workspace = true+smol.workspace = true+thiserror.workspace = true+url = { workspace = true, features = ["serde"] }++[dev-dependencies]+http_client = { workspace = true, features = ["test-support"] }diff --git a/tooling/crates_io_client/LICENSE-APACHE b/tooling/crates_io_client/LICENSE-APACHEnew file mode 120000index 00000000000..1cd601d0a3a--- /dev/null+++ b/tooling/crates_io_client/LICENSE-APACHE@@ -0,0 +1 @@+../../LICENSE-APACHE\ No newline at end of filediff --git a/tooling/crates_io_client/src/crates_io_client.rs b/tooling/crates_io_client/src/crates_io_client.rsnew file mode 100644index 00000000000..ec158c59054--- /dev/null+++ b/tooling/crates_io_client/src/crates_io_client.rs@@ -0,0 +1,81 @@+mod real_crates_io_client;+#[cfg(any(test, feature = "test-support"))]+pub mod simulated_crates_io_client;+mod types;++#[cfg(test)]+mod tests;++use async_trait::async_trait;+use http_client::StatusCode;+use itertools::Itertools as _;+use serde::Deserialize;++pub use real_crates_io_client::RealCratesIoClient;+pub use types::*;++pub type Result<T, E = CratesIoError> = std::result::Result<T, E>;++#[async_trait]+pub trait CratesIoClient: Send + Sync {+ async fn get_site_metadata(&self) -> Result<SiteMetadata>;+}++#[derive(Debug, thiserror::Error)]+pub enum CratesIoError {+ /// The token is invalid, or lacks the scope or crate ownership the+ /// request requires.+ #[error("crates.io denied permission ({status}): {message}")]+ PermissionDenied { status: StatusCode, message: String },+ #[error("crates.io could not find the requested resource: {message}")]+ NotFound { message: String },+ #[error("crates.io request failed with {status}: {message}")]+ Upstream { status: StatusCode, message: String },+ /// The request could not be sent or its response could not be read.+ #[error(transparent)]+ Request(#[from] anyhow::Error),+}++impl CratesIoError {+ fn from_response(status: StatusCode, body: &str) -> Self {+ #[derive(Deserialize)]+ struct ErrorResponse {+ errors: Vec<ErrorDetail>,+ }++ #[derive(Deserialize)]+ struct ErrorDetail {+ detail: String,+ }++ let message = match serde_json::from_str::<ErrorResponse>(body) {+ Ok(response) => response+ .errors+ .into_iter()+ .map(|error| error.detail)+ .join("; "),+ Err(_) => body.to_owned(),+ };++ match status {+ StatusCode::UNAUTHORIZED | StatusCode::FORBIDDEN => {+ Self::PermissionDenied { status, message }+ }+ StatusCode::NOT_FOUND => Self::NotFound { message },+ _ => Self::Upstream { status, message },+ }+ }+}++#[derive(Clone)]+pub struct ApiToken(String);++impl ApiToken {+ pub fn new(value: impl Into<String>) -> Self {+ Self(value.into())+ }++ pub(crate) fn value(&self) -> &str {+ &self.0+ }+}diff --git a/tooling/crates_io_client/src/real_crates_io_client.rs b/tooling/crates_io_client/src/real_crates_io_client.rsnew file mode 100644index 00000000000..afcc22f78a1--- /dev/null+++ b/tooling/crates_io_client/src/real_crates_io_client.rs@@ -0,0 +1,127 @@+use std::{+ sync::Arc,+ time::{Duration, Instant},+};++use anyhow::Context as _;+use async_trait::async_trait;+use futures::AsyncReadExt as _;+use http_client::{AsyncBody, HttpClient, Method, Request, Response, http::header};+use parking_lot::Mutex;+use serde::de::DeserializeOwned;+use url::Url;++use crate::*;++const BASE_URL: &str = "https://crates.io/api/v1";+// User agent and request rate requirements: https://crates.io/data-access+const USER_AGENT: &str = "Zed GPUI releases (https://github.com/zed-industries/zed)";+pub(crate) const MIN_REQUEST_INTERVAL: Duration = Duration::from_secs(1);++/// Reserving the send time before waiting makes concurrent requests queue up+/// behind each other instead of all waking at the same instant.+pub(crate) struct RequestThrottle {+ next_request_at: Mutex<Option<Instant>>,+}++impl RequestThrottle {+ pub(crate) fn new() -> Self {+ Self {+ next_request_at: Mutex::new(None),+ }+ }++ pub(crate) fn reserve(&self, now: Instant) -> Instant {+ let mut next_request_at = self.next_request_at.lock();+ let send_at = next_request_at.map_or(now, |next_request_at| next_request_at.max(now));+ *next_request_at = Some(send_at + MIN_REQUEST_INTERVAL);+ send_at+ }+}++/// The request rate limit is enforced per client, so share a single client.+pub struct RealCratesIoClient {+ http_client: Arc<dyn HttpClient>,+ api_token: ApiToken,+ throttle: RequestThrottle,+}++impl RealCratesIoClient {+ pub fn new(http_client: Arc<dyn HttpClient>, api_token: ApiToken) -> Self {+ Self {+ http_client,+ api_token,+ throttle: RequestThrottle::new(),+ }+ }++ fn url(path: &str) -> Result<Url> {+ Ok(Url::parse(&format!("{BASE_URL}{path}")).context("Invalid crates.io URL")?)+ }++ async fn request_response(+ &self,+ method: Method,+ url: Url,+ body: Option<Vec<u8>>,+ ) -> Result<Response<AsyncBody>> {+ let mut request = Request::builder()+ .method(method)+ .uri(url.as_str())+ .header(header::USER_AGENT, USER_AGENT)+ .header(header::ACCEPT, "application/json")+ .header(header::AUTHORIZATION, self.api_token.value());+ if body.is_some() {+ request = request.header(header::CONTENT_TYPE, "application/json");+ }+ let request = request+ .body(body.map_or_else(AsyncBody::empty, AsyncBody::from))+ .context("Failed to build crates.io request")?;++ let send_at = self.throttle.reserve(Instant::now());+ smol::Timer::at(send_at).await;++ let mut response = self+ .http_client+ .send(request)+ .await+ .with_context(|| format!("Failed to send crates.io request to {url}"))?;++ if !response.status().is_success() {+ let mut body = String::new();+ response+ .body_mut()+ .read_to_string(&mut body)+ .await+ .with_context(|| format!("Failed to read crates.io response from {url}"))?;+ return Err(CratesIoError::from_response(response.status(), &body));+ }++ Ok(response)+ }++ async fn request_json<T: DeserializeOwned>(+ &self,+ method: Method,+ url: Url,+ body: Option<Vec<u8>>,+ ) -> Result<T> {+ let mut response = self.request_response(method, url.clone(), body).await?;+ let mut body = Vec::new();+ response+ .body_mut()+ .read_to_end(&mut body)+ .await+ .with_context(|| format!("Failed to read crates.io response from {url}"))?;+ Ok(serde_json::from_slice(&body)+ .with_context(|| format!("Failed to parse crates.io response from {url}"))?)+ }+}++#[async_trait]+impl CratesIoClient for RealCratesIoClient {+ async fn get_site_metadata(&self) -> Result<SiteMetadata> {+ self.request_json(Method::GET, Self::url("/site_metadata")?, None)+ .await+ }+}diff --git a/tooling/crates_io_client/src/simulated_crates_io_client.rs b/tooling/crates_io_client/src/simulated_crates_io_client.rsnew file mode 100644index 00000000000..9268455b0ea--- /dev/null+++ b/tooling/crates_io_client/src/simulated_crates_io_client.rs@@ -0,0 +1,42 @@+use std::sync::Arc;++use async_trait::async_trait;+use parking_lot::Mutex;+use url::Url;++use crate::*;++pub struct SimulatedCratesIoClient {+ state: Mutex<CratesIoState>,+}++struct CratesIoState {+ site_metadata: SiteMetadata,+}++impl SimulatedCratesIoClient {+ pub fn new() -> Arc<Self> {+ Arc::new(Self {+ state: Mutex::new(CratesIoState {+ site_metadata: SiteMetadata {+ deployed_sha: "unknown".to_owned(),+ commit: "unknown".to_owned(),+ cdn_base: Url::parse("https://static.example.com").expect("test URL is valid"),+ read_only: false,+ banner_message: None,+ },+ }),+ })+ }++ pub fn test_site_metadata(&self, site_metadata: SiteMetadata) {+ self.state.lock().site_metadata = site_metadata;+ }+}++#[async_trait]+impl CratesIoClient for SimulatedCratesIoClient {+ async fn get_site_metadata(&self) -> Result<SiteMetadata> {+ Ok(self.state.lock().site_metadata.clone())+ }+}diff --git a/tooling/crates_io_client/src/tests.rs b/tooling/crates_io_client/src/tests.rsnew file mode 100644index 00000000000..365c4d5e982--- /dev/null+++ b/tooling/crates_io_client/src/tests.rs@@ -0,0 +1,113 @@+use std::{+ sync::Arc,+ time::{Duration, Instant},+};++use http_client::{+ AsyncBody, FakeHttpClient, HttpClient, Response, StatusCode,+ http::{HeaderMap, header},+};+use parking_lot::Mutex;+use serde_json::{Value, json};++use crate::{+ real_crates_io_client::{MIN_REQUEST_INTERVAL, RequestThrottle},+ *,+};++struct RecordedRequest {+ headers: HeaderMap,+}++/// Returns a fake HTTP client that records each request and answers it with+/// the status and JSON body `respond` returns for the request's URI.+fn recording_fake_http_client(+ respond: impl Fn(&str) -> (StatusCode, Value) + Send + Sync + 'static,+) -> (Arc<dyn HttpClient>, Arc<Mutex<Vec<RecordedRequest>>>) {+ let requests = Arc::new(Mutex::new(Vec::new()));+ let respond = Arc::new(respond);+ let fake_http_client = FakeHttpClient::create({+ let requests = requests.clone();+ move |request| {+ let requests = requests.clone();+ let respond = respond.clone();+ async move {+ let (status, response_body) = respond(&request.uri().to_string());+ requests.lock().push(RecordedRequest {+ headers: request.headers().clone(),+ });+ Ok(Response::builder()+ .status(status)+ .body(AsyncBody::from(response_body.to_string()))?)+ }+ }+ });+ (fake_http_client, requests)+}++#[test]+fn test_requests_identify_zed_and_authenticate_with_api_token() {+ let (fake_http_client, requests) = recording_fake_http_client(|_| {+ let site_metadata = json!({+ "deployed_sha": "abc",+ "commit": "abc",+ "cdn_base": "https://static.crates.io",+ "read_only": false,+ });+ (StatusCode::OK, site_metadata)+ });+ let client = RealCratesIoClient::new(fake_http_client, ApiToken::new("secret"));++ smol::block_on(client.get_site_metadata()).unwrap();++ let requests = requests.lock();+ assert_eq!(+ requests[0].headers[header::USER_AGENT],+ "Zed GPUI releases (https://github.com/zed-industries/zed)"+ );+ assert_eq!(requests[0].headers[header::AUTHORIZATION], "secret");+}++#[test]+fn test_error_responses_distinguish_permission_and_upstream_failures() {+ let error_for = |status: StatusCode| {+ let (fake_http_client, _requests) = recording_fake_http_client(move |_| {+ let errors = json!({ "errors": [{ "detail": "first" }, { "detail": "second" }] });+ (status, errors)+ });+ let client = RealCratesIoClient::new(fake_http_client, ApiToken::new("secret"));+ smol::block_on(client.get_site_metadata()).unwrap_err()+ };++ assert!(matches!(+ error_for(StatusCode::FORBIDDEN),+ CratesIoError::PermissionDenied { status: StatusCode::FORBIDDEN, message } if message == "first; second"+ ));+ assert!(matches!(+ error_for(StatusCode::UNAUTHORIZED),+ CratesIoError::PermissionDenied { .. }+ ));+ assert!(matches!(+ error_for(StatusCode::NOT_FOUND),+ CratesIoError::NotFound { .. }+ ));+ assert!(matches!(+ error_for(StatusCode::SERVICE_UNAVAILABLE),+ CratesIoError::Upstream {+ status: StatusCode::SERVICE_UNAVAILABLE,+ ..+ }+ ));+}++#[test]+fn test_throttle_spaces_requests_by_minimum_interval() {+ let throttle = RequestThrottle::new();+ let start = Instant::now();++ assert_eq!(throttle.reserve(start), start);+ assert_eq!(throttle.reserve(start), start + MIN_REQUEST_INTERVAL);+ assert_eq!(throttle.reserve(start), start + 2 * MIN_REQUEST_INTERVAL);+ let later = start + Duration::from_secs(10);+ assert_eq!(throttle.reserve(later), later);+}diff --git a/tooling/crates_io_client/src/types.rs b/tooling/crates_io_client/src/types.rsnew file mode 100644index 00000000000..58fda9e2a7c--- /dev/null+++ b/tooling/crates_io_client/src/types.rs@@ -0,0 +1,12 @@+use serde::{Deserialize, Serialize};+use url::Url;++#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]+pub struct SiteMetadata {+ pub deployed_sha: String,+ pub commit: String,+ pub cdn_base: Url,+ pub read_only: bool,+ #[serde(default)]+ pub banner_message: Option<String>,+}