Skip to content

armgit / zed

Public Imported from GitHub
0

Code at the speed of thought – Zed is a high-performance, multiplayer code editor from the creators of Atom and Tree-sitter.

crates_io_client: Add crates.io API client (#66437)

Finn Eversccb25cbc6376
diff --git a/Cargo.lock b/Cargo.lock
index a069b149791..ed76ba16512 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -4098,6 +4098,23 @@ dependencies = [
"zstd",
]
+[[package]]
+name = "crates_io_client"
+version = "0.1.0"
+dependencies = [
+ "anyhow",
+ "async-trait",
+ "futures 0.3.34",
+ "http_client",
+ "itertools 0.14.0",
+ "parking_lot",
+ "serde",
+ "serde_json",
+ "smol",
+ "thiserror 2.0.17",
+ "url",
+]
+
[[package]]
name = "crc"
version = "3.3.0"
diff --git a/Cargo.toml b/Cargo.toml
index 5b1f5473e6d..b612bdf8744 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -268,6 +268,7 @@ members = [
#
"tooling/compliance",
+ "tooling/crates_io_client",
"tooling/perf",
"tooling/xtask",
]
diff --git a/tooling/crates_io_client/Cargo.toml b/tooling/crates_io_client/Cargo.toml
new file mode 100644
index 00000000000..3928072748f
--- /dev/null
+++ b/tooling/crates_io_client/Cargo.toml
@@ -0,0 +1,33 @@
+[package]
+name = "crates_io_client"
+version = "0.1.0"
+edition.workspace = true
+publish.workspace = true
+license = "Apache-2.0"
+description = "crates.io <https://crates.io> API client"
+
+[lints]
+workspace = true
+
+[features]
+test-support = []
+
+[lib]
+path = "src/crates_io_client.rs"
+doctest = false
+
+[dependencies]
+anyhow.workspace = true
+async-trait.workspace = true
+futures.workspace = true
+http_client.workspace = true
+itertools.workspace = true
+parking_lot.workspace = true
+serde = { workspace = true, features = ["derive"] }
+serde_json.workspace = true
+smol.workspace = true
+thiserror.workspace = true
+url = { workspace = true, features = ["serde"] }
+
+[dev-dependencies]
+http_client = { workspace = true, features = ["test-support"] }
diff --git a/tooling/crates_io_client/LICENSE-APACHE b/tooling/crates_io_client/LICENSE-APACHE
new file mode 120000
index 00000000000..1cd601d0a3a
--- /dev/null
+++ b/tooling/crates_io_client/LICENSE-APACHE
@@ -0,0 +1 @@
+../../LICENSE-APACHE
\ No newline at end of file
diff --git a/tooling/crates_io_client/src/crates_io_client.rs b/tooling/crates_io_client/src/crates_io_client.rs
new file mode 100644
index 00000000000..ec158c59054
--- /dev/null
+++ b/tooling/crates_io_client/src/crates_io_client.rs
@@ -0,0 +1,81 @@
+mod real_crates_io_client;
+#[cfg(any(test, feature = "test-support"))]
+pub mod simulated_crates_io_client;
+mod types;
+
+#[cfg(test)]
+mod tests;
+
+use async_trait::async_trait;
+use http_client::StatusCode;
+use itertools::Itertools as _;
+use serde::Deserialize;
+
+pub use real_crates_io_client::RealCratesIoClient;
+pub use types::*;
+
+pub type Result<T, E = CratesIoError> = std::result::Result<T, E>;
+
+#[async_trait]
+pub trait CratesIoClient: Send + Sync {
+ async fn get_site_metadata(&self) -> Result<SiteMetadata>;
+}
+
+#[derive(Debug, thiserror::Error)]
+pub enum CratesIoError {
+ /// The token is invalid, or lacks the scope or crate ownership the
+ /// request requires.
+ #[error("crates.io denied permission ({status}): {message}")]
+ PermissionDenied { status: StatusCode, message: String },
+ #[error("crates.io could not find the requested resource: {message}")]
+ NotFound { message: String },
+ #[error("crates.io request failed with {status}: {message}")]
+ Upstream { status: StatusCode, message: String },
+ /// The request could not be sent or its response could not be read.
+ #[error(transparent)]
+ Request(#[from] anyhow::Error),
+}
+
+impl CratesIoError {
+ fn from_response(status: StatusCode, body: &str) -> Self {
+ #[derive(Deserialize)]
+ struct ErrorResponse {
+ errors: Vec<ErrorDetail>,
+ }
+
+ #[derive(Deserialize)]
+ struct ErrorDetail {
+ detail: String,
+ }
+
+ let message = match serde_json::from_str::<ErrorResponse>(body) {
+ Ok(response) => response
+ .errors
+ .into_iter()
+ .map(|error| error.detail)
+ .join("; "),
+ Err(_) => body.to_owned(),
+ };
+
+ match status {
+ StatusCode::UNAUTHORIZED | StatusCode::FORBIDDEN => {
+ Self::PermissionDenied { status, message }
+ }
+ StatusCode::NOT_FOUND => Self::NotFound { message },
+ _ => Self::Upstream { status, message },
+ }
+ }
+}
+
+#[derive(Clone)]
+pub struct ApiToken(String);
+
+impl ApiToken {
+ pub fn new(value: impl Into<String>) -> Self {
+ Self(value.into())
+ }
+
+ pub(crate) fn value(&self) -> &str {
+ &self.0
+ }
+}
diff --git a/tooling/crates_io_client/src/real_crates_io_client.rs b/tooling/crates_io_client/src/real_crates_io_client.rs
new file mode 100644
index 00000000000..afcc22f78a1
--- /dev/null
+++ b/tooling/crates_io_client/src/real_crates_io_client.rs
@@ -0,0 +1,127 @@
+use std::{
+ sync::Arc,
+ time::{Duration, Instant},
+};
+
+use anyhow::Context as _;
+use async_trait::async_trait;
+use futures::AsyncReadExt as _;
+use http_client::{AsyncBody, HttpClient, Method, Request, Response, http::header};
+use parking_lot::Mutex;
+use serde::de::DeserializeOwned;
+use url::Url;
+
+use crate::*;
+
+const BASE_URL: &str = "https://crates.io/api/v1";
+// User agent and request rate requirements: https://crates.io/data-access
+const USER_AGENT: &str = "Zed GPUI releases (https://github.com/zed-industries/zed)";
+pub(crate) const MIN_REQUEST_INTERVAL: Duration = Duration::from_secs(1);
+
+/// Reserving the send time before waiting makes concurrent requests queue up
+/// behind each other instead of all waking at the same instant.
+pub(crate) struct RequestThrottle {
+ next_request_at: Mutex<Option<Instant>>,
+}
+
+impl RequestThrottle {
+ pub(crate) fn new() -> Self {
+ Self {
+ next_request_at: Mutex::new(None),
+ }
+ }
+
+ pub(crate) fn reserve(&self, now: Instant) -> Instant {
+ let mut next_request_at = self.next_request_at.lock();
+ let send_at = next_request_at.map_or(now, |next_request_at| next_request_at.max(now));
+ *next_request_at = Some(send_at + MIN_REQUEST_INTERVAL);
+ send_at
+ }
+}
+
+/// The request rate limit is enforced per client, so share a single client.
+pub struct RealCratesIoClient {
+ http_client: Arc<dyn HttpClient>,
+ api_token: ApiToken,
+ throttle: RequestThrottle,
+}
+
+impl RealCratesIoClient {
+ pub fn new(http_client: Arc<dyn HttpClient>, api_token: ApiToken) -> Self {
+ Self {
+ http_client,
+ api_token,
+ throttle: RequestThrottle::new(),
+ }
+ }
+
+ fn url(path: &str) -> Result<Url> {
+ Ok(Url::parse(&format!("{BASE_URL}{path}")).context("Invalid crates.io URL")?)
+ }
+
+ async fn request_response(
+ &self,
+ method: Method,
+ url: Url,
+ body: Option<Vec<u8>>,
+ ) -> Result<Response<AsyncBody>> {
+ let mut request = Request::builder()
+ .method(method)
+ .uri(url.as_str())
+ .header(header::USER_AGENT, USER_AGENT)
+ .header(header::ACCEPT, "application/json")
+ .header(header::AUTHORIZATION, self.api_token.value());
+ if body.is_some() {
+ request = request.header(header::CONTENT_TYPE, "application/json");
+ }
+ let request = request
+ .body(body.map_or_else(AsyncBody::empty, AsyncBody::from))
+ .context("Failed to build crates.io request")?;
+
+ let send_at = self.throttle.reserve(Instant::now());
+ smol::Timer::at(send_at).await;
+
+ let mut response = self
+ .http_client
+ .send(request)
+ .await
+ .with_context(|| format!("Failed to send crates.io request to {url}"))?;
+
+ if !response.status().is_success() {
+ let mut body = String::new();
+ response
+ .body_mut()
+ .read_to_string(&mut body)
+ .await
+ .with_context(|| format!("Failed to read crates.io response from {url}"))?;
+ return Err(CratesIoError::from_response(response.status(), &body));
+ }
+
+ Ok(response)
+ }
+
+ async fn request_json<T: DeserializeOwned>(
+ &self,
+ method: Method,
+ url: Url,
+ body: Option<Vec<u8>>,
+ ) -> Result<T> {
+ let mut response = self.request_response(method, url.clone(), body).await?;
+ let mut body = Vec::new();
+ response
+ .body_mut()
+ .read_to_end(&mut body)
+ .await
+ .with_context(|| format!("Failed to read crates.io response from {url}"))?;
+ Ok(serde_json::from_slice(&body)
+ .with_context(|| format!("Failed to parse crates.io response from {url}"))?)
+ }
+}
+
+#[async_trait]
+impl CratesIoClient for RealCratesIoClient {
+ async fn get_site_metadata(&self) -> Result<SiteMetadata> {
+ self.request_json(Method::GET, Self::url("/site_metadata")?, None)
+ .await
+ }
+}
diff --git a/tooling/crates_io_client/src/simulated_crates_io_client.rs b/tooling/crates_io_client/src/simulated_crates_io_client.rs
new file mode 100644
index 00000000000..9268455b0ea
--- /dev/null
+++ b/tooling/crates_io_client/src/simulated_crates_io_client.rs
@@ -0,0 +1,42 @@
+use std::sync::Arc;
+
+use async_trait::async_trait;
+use parking_lot::Mutex;
+use url::Url;
+
+use crate::*;
+
+pub struct SimulatedCratesIoClient {
+ state: Mutex<CratesIoState>,
+}
+
+struct CratesIoState {
+ site_metadata: SiteMetadata,
+}
+
+impl SimulatedCratesIoClient {
+ pub fn new() -> Arc<Self> {
+ Arc::new(Self {
+ state: Mutex::new(CratesIoState {
+ site_metadata: SiteMetadata {
+ deployed_sha: "unknown".to_owned(),
+ commit: "unknown".to_owned(),
+ cdn_base: Url::parse("https://static.example.com").expect("test URL is valid"),
+ read_only: false,
+ banner_message: None,
+ },
+ }),
+ })
+ }
+
+ pub fn test_site_metadata(&self, site_metadata: SiteMetadata) {
+ self.state.lock().site_metadata = site_metadata;
+ }
+}
+
+#[async_trait]
+impl CratesIoClient for SimulatedCratesIoClient {
+ async fn get_site_metadata(&self) -> Result<SiteMetadata> {
+ Ok(self.state.lock().site_metadata.clone())
+ }
+}
diff --git a/tooling/crates_io_client/src/tests.rs b/tooling/crates_io_client/src/tests.rs
new file mode 100644
index 00000000000..365c4d5e982
--- /dev/null
+++ b/tooling/crates_io_client/src/tests.rs
@@ -0,0 +1,113 @@
+use std::{
+ sync::Arc,
+ time::{Duration, Instant},
+};
+
+use http_client::{
+ AsyncBody, FakeHttpClient, HttpClient, Response, StatusCode,
+ http::{HeaderMap, header},
+};
+use parking_lot::Mutex;
+use serde_json::{Value, json};
+
+use crate::{
+ real_crates_io_client::{MIN_REQUEST_INTERVAL, RequestThrottle},
+ *,
+};
+
+struct RecordedRequest {
+ headers: HeaderMap,
+}
+
+/// Returns a fake HTTP client that records each request and answers it with
+/// the status and JSON body `respond` returns for the request's URI.
+fn recording_fake_http_client(
+ respond: impl Fn(&str) -> (StatusCode, Value) + Send + Sync + 'static,
+) -> (Arc<dyn HttpClient>, Arc<Mutex<Vec<RecordedRequest>>>) {
+ let requests = Arc::new(Mutex::new(Vec::new()));
+ let respond = Arc::new(respond);
+ let fake_http_client = FakeHttpClient::create({
+ let requests = requests.clone();
+ move |request| {
+ let requests = requests.clone();
+ let respond = respond.clone();
+ async move {
+ let (status, response_body) = respond(&request.uri().to_string());
+ requests.lock().push(RecordedRequest {
+ headers: request.headers().clone(),
+ });
+ Ok(Response::builder()
+ .status(status)
+ .body(AsyncBody::from(response_body.to_string()))?)
+ }
+ }
+ });
+ (fake_http_client, requests)
+}
+
+#[test]
+fn test_requests_identify_zed_and_authenticate_with_api_token() {
+ let (fake_http_client, requests) = recording_fake_http_client(|_| {
+ let site_metadata = json!({
+ "deployed_sha": "abc",
+ "commit": "abc",
+ "cdn_base": "https://static.crates.io",
+ "read_only": false,
+ });
+ (StatusCode::OK, site_metadata)
+ });
+ let client = RealCratesIoClient::new(fake_http_client, ApiToken::new("secret"));
+
+ smol::block_on(client.get_site_metadata()).unwrap();
+
+ let requests = requests.lock();
+ assert_eq!(
+ requests[0].headers[header::USER_AGENT],
+ "Zed GPUI releases (https://github.com/zed-industries/zed)"
+ );
+ assert_eq!(requests[0].headers[header::AUTHORIZATION], "secret");
+}
+
+#[test]
+fn test_error_responses_distinguish_permission_and_upstream_failures() {
+ let error_for = |status: StatusCode| {
+ let (fake_http_client, _requests) = recording_fake_http_client(move |_| {
+ let errors = json!({ "errors": [{ "detail": "first" }, { "detail": "second" }] });
+ (status, errors)
+ });
+ let client = RealCratesIoClient::new(fake_http_client, ApiToken::new("secret"));
+ smol::block_on(client.get_site_metadata()).unwrap_err()
+ };
+
+ assert!(matches!(
+ error_for(StatusCode::FORBIDDEN),
+ CratesIoError::PermissionDenied { status: StatusCode::FORBIDDEN, message } if message == "first; second"
+ ));
+ assert!(matches!(
+ error_for(StatusCode::UNAUTHORIZED),
+ CratesIoError::PermissionDenied { .. }
+ ));
+ assert!(matches!(
+ error_for(StatusCode::NOT_FOUND),
+ CratesIoError::NotFound { .. }
+ ));
+ assert!(matches!(
+ error_for(StatusCode::SERVICE_UNAVAILABLE),
+ CratesIoError::Upstream {
+ status: StatusCode::SERVICE_UNAVAILABLE,
+ ..
+ }
+ ));
+}
+
+#[test]
+fn test_throttle_spaces_requests_by_minimum_interval() {
+ let throttle = RequestThrottle::new();
+ let start = Instant::now();
+
+ assert_eq!(throttle.reserve(start), start);
+ assert_eq!(throttle.reserve(start), start + MIN_REQUEST_INTERVAL);
+ assert_eq!(throttle.reserve(start), start + 2 * MIN_REQUEST_INTERVAL);
+ let later = start + Duration::from_secs(10);
+ assert_eq!(throttle.reserve(later), later);
+}
diff --git a/tooling/crates_io_client/src/types.rs b/tooling/crates_io_client/src/types.rs
new file mode 100644
index 00000000000..58fda9e2a7c
--- /dev/null
+++ b/tooling/crates_io_client/src/types.rs
@@ -0,0 +1,12 @@
+use serde::{Deserialize, Serialize};
+use url::Url;
+
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+pub struct SiteMetadata {
+ pub deployed_sha: String,
+ pub commit: String,
+ pub cdn_base: Url,
+ pub read_only: bool,
+ #[serde(default)]
+ pub banner_message: Option<String>,
+}