Code at the speed of thought – Zed is a high-performance, multiplayer code editor from the creators of Atom and Tree-sitter.
crates_io_client: Add endpoints for listing, getting, and updating crates (#66438)
Finn Evers4ebf3eac14d5
diff --git a/Cargo.lock b/Cargo.lockindex ed76ba16512..6695899a1f7 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -4104,10 +4104,12 @@ version = "0.1.0"dependencies = ["anyhow","async-trait",+ "chrono","futures 0.3.34","http_client","itertools 0.14.0","parking_lot",+ "semver","serde","serde_json","smol",diff --git a/tooling/crates_io_client/Cargo.toml b/tooling/crates_io_client/Cargo.tomlindex 3928072748f..ea72a581839 100644--- a/tooling/crates_io_client/Cargo.toml+++ b/tooling/crates_io_client/Cargo.toml@@ -19,11 +19,13 @@ doctest = false[dependencies]anyhow.workspace = trueasync-trait.workspace = true+chrono.workspace = truefutures.workspace = truehttp_client.workspace = trueitertools.workspace = trueparking_lot.workspace = trueserde = { workspace = true, features = ["derive"] }+semver.workspace = trueserde_json.workspace = truesmol.workspace = truethiserror.workspace = truediff --git a/tooling/crates_io_client/src/crates_io_client.rs b/tooling/crates_io_client/src/crates_io_client.rsindex ec158c59054..1b6e627dc44 100644--- a/tooling/crates_io_client/src/crates_io_client.rs+++ b/tooling/crates_io_client/src/crates_io_client.rs@@ -19,6 +19,16 @@ pub type Result<T, E = CratesIoError> = std::result::Result<T, E>;#[async_trait]pub trait CratesIoClient: Send + Sync {async fn get_site_metadata(&self) -> Result<SiteMetadata>;++ /// Lists every crate matching the query, requesting one page of 100 crates+ /// per second.+ async fn list_crates(&self, query: &CrateListQuery) -> Result<Vec<Crate>>;++ async fn get_crate(&self, name: &CrateName) -> Result<CrateDetails>;++ /// Requires a token with the `trusted-publishing` endpoint scope that+ /// belongs to an owner of the crate.+ async fn update_crate(&self, name: &CrateName, update: &CrateUpdate) -> Result<Crate>;}#[derive(Debug, thiserror::Error)]diff --git a/tooling/crates_io_client/src/real_crates_io_client.rs b/tooling/crates_io_client/src/real_crates_io_client.rsindex afcc22f78a1..f282a3f9c93 100644--- a/tooling/crates_io_client/src/real_crates_io_client.rs+++ b/tooling/crates_io_client/src/real_crates_io_client.rs@@ -8,7 +8,7 @@ use async_trait::async_trait;use futures::AsyncReadExt as _;use http_client::{AsyncBody, HttpClient, Method, Request, Response, http::header};use parking_lot::Mutex;-use serde::de::DeserializeOwned;+use serde::{Deserialize, Serialize, de::DeserializeOwned};use url::Url;use crate::*;@@ -17,6 +17,7 @@ const BASE_URL: &str = "https://crates.io/api/v1";// User agent and request rate requirements: https://crates.io/data-accessconst USER_AGENT: &str = "Zed GPUI releases (https://github.com/zed-industries/zed)";pub(crate) const MIN_REQUEST_INTERVAL: Duration = Duration::from_secs(1);+const MAX_PER_PAGE: u32 = 100;/// Reserving the send time before waiting makes concurrent requests queue up/// behind each other instead of all waking at the same instant.@@ -59,6 +60,14 @@ impl RealCratesIoClient {Ok(Url::parse(&format!("{BASE_URL}{path}")).context("Invalid crates.io URL")?)}+ fn crate_url(name: &CrateName) -> Result<Url> {+ let mut url = Self::url("/crates")?;+ url.path_segments_mut()+ .map_err(|()| anyhow::anyhow!("crates.io URL cannot have path segments"))?+ .push(name.as_str());+ Ok(url)+ }+async fn request_response(&self,method: Method,@@ -124,4 +133,92 @@ impl CratesIoClient for RealCratesIoClient {self.request_json(Method::GET, Self::url("/site_metadata")?, None).await}++ async fn list_crates(&self, query: &CrateListQuery) -> Result<Vec<Crate>> {+ #[derive(Deserialize)]+ struct CrateListResponse {+ crates: Vec<Crate>,+ meta: CrateListMeta,+ }++ #[derive(Deserialize)]+ struct CrateListMeta {+ /// A query string for the next page, which preserves the filters.+ next_page: Option<String>,+ }++ let mut url = Self::url("/crates")?;+ {+ let mut pairs = url.query_pairs_mut();+ pairs.append_pair("per_page", &MAX_PER_PAGE.to_string());+ if let Some(search) = &query.search {+ pairs.append_pair("q", search);+ }+ if let Some(category) = &query.category {+ pairs.append_pair("category", category);+ }+ if let Some(keyword) = &query.keyword {+ pairs.append_pair("keyword", keyword);+ }+ if let Some(user_id) = query.user_id {+ pairs.append_pair("user_id", &user_id.0.to_string());+ }+ if query.exclude_yanked {+ pairs.append_pair("include_yanked", "no");+ }+ }++ let mut crates = Vec::new();+ loop {+ let mut page: CrateListResponse = self.request_json(Method::GET, url, None).await?;+ let is_last_page = page.crates.is_empty();+ crates.append(&mut page.crates);++ match page.meta.next_page {+ Some(next_page) if !is_last_page => {+ url = Self::url(&format!("/crates{next_page}"))?;+ }+ _ => return Ok(crates),+ }+ }+ }++ async fn get_crate(&self, name: &CrateName) -> Result<CrateDetails> {+ #[derive(Deserialize)]+ struct CrateResponse {+ #[serde(rename = "crate")]+ krate: Crate,+ versions: Option<Vec<Version>>,+ }++ let mut url = Self::crate_url(name)?;+ url.query_pairs_mut()+ .append_pair("include", "versions,keywords,categories");+ let response: CrateResponse = self.request_json(Method::GET, url, None).await?;+ Ok(CrateDetails {+ krate: response.krate,+ versions: response.versions.unwrap_or_default(),+ })+ }++ async fn update_crate(&self, name: &CrateName, update: &CrateUpdate) -> Result<Crate> {+ #[derive(Serialize)]+ struct UpdateRequest<'a> {+ #[serde(rename = "crate")]+ krate: &'a CrateUpdate,+ }++ #[derive(Deserialize)]+ struct UpdateResponse {+ #[serde(rename = "crate")]+ krate: Crate,+ }++ let body = serde_json::to_vec(&UpdateRequest { krate: update })+ .context("Failed to serialize crate update")?;+ let response: UpdateResponse = self+ .request_json(Method::PATCH, Self::crate_url(name)?, Some(body))+ .await?;+ Ok(response.krate)+ }}diff --git a/tooling/crates_io_client/src/simulated_crates_io_client.rs b/tooling/crates_io_client/src/simulated_crates_io_client.rsindex 9268455b0ea..0f0afd3ab6e 100644--- a/tooling/crates_io_client/src/simulated_crates_io_client.rs+++ b/tooling/crates_io_client/src/simulated_crates_io_client.rs@@ -1,6 +1,8 @@-use std::sync::Arc;+use std::{collections::BTreeMap, sync::Arc};use async_trait::async_trait;+use chrono::{DateTime, Utc};+use http_client::StatusCode;use parking_lot::Mutex;use url::Url;@@ -12,9 +14,47 @@ pub struct SimulatedCratesIoClient {struct CratesIoState {site_metadata: SiteMetadata,+ token_user: UserId,+ crates: BTreeMap<CrateName, SimulatedCrate>,+ latest_id: i32,+}++struct SimulatedCrate {+ krate: Crate,+ owners: Vec<UserId>,+ versions: Vec<Version>,+}++impl CratesIoState {+ fn next_id(&mut self) -> i32 {+ self.latest_id += 1;+ self.latest_id+ }++ fn simulated_crate(&mut self, name: &CrateName) -> Result<&mut SimulatedCrate> {+ self.crates+ .get_mut(name)+ .ok_or_else(|| CratesIoError::NotFound {+ message: format!("crate `{name}` does not exist"),+ })+ }+}++/// Keyword and category IDs are only part of single-crate responses.+fn listed(krate: &Crate) -> Crate {+ Crate {+ keywords: None,+ categories: None,+ ..krate.clone()+ }+}++fn test_timestamp() -> DateTime<Utc> {+ DateTime::from_timestamp(1_767_225_600, 0).unwrap_or_default()}impl SimulatedCratesIoClient {+ /// Creates a registry whose API token belongs to [`Self::token_user`].pub fn new() -> Arc<Self> {Arc::new(Self {state: Mutex::new(CratesIoState {@@ -25,13 +65,100 @@ impl SimulatedCratesIoClient {read_only: false,banner_message: None,},+ token_user: UserId(1),+ crates: BTreeMap::new(),+ latest_id: 1,}),})}+ pub fn token_user(&self) -> UserId {+ self.state.lock().token_user+ }++ pub fn test_user(&self) -> UserId {+ UserId(self.state.lock().next_id())+ }+pub fn test_site_metadata(&self, site_metadata: SiteMetadata) {self.state.lock().site_metadata = site_metadata;}++ /// Adds a crate without versions, replacing any crate with the same name.+ pub fn test_crate(&self, name: &str, owner: UserId) -> CrateName {+ let name = CrateName::new(name);+ self.state.lock().crates.insert(+ name.clone(),+ SimulatedCrate {+ krate: Crate {+ name: name.clone(),+ description: None,+ homepage: None,+ documentation: None,+ repository: None,+ created_at: test_timestamp(),+ updated_at: test_timestamp(),+ downloads: 0,+ recent_downloads: Some(0),+ default_version: None,+ num_versions: 0,+ yanked: false,+ trustpub_only: false,+ keywords: Some(Vec::new()),+ categories: Some(Vec::new()),+ },+ owners: vec![owner],+ versions: Vec::new(),+ },+ );+ name+ }++ /// Changes crate fields that have no dedicated helper, such as its+ /// keywords or download counts.+ pub fn update_test_crate(+ &self,+ name: &CrateName,+ update: impl FnOnce(&mut Crate),+ ) -> Result<()> {+ let mut state = self.state.lock();+ update(&mut state.simulated_crate(name)?.krate);+ Ok(())+ }++ /// Publishes a version, which becomes the crate's default version.+ pub fn test_version(&self, name: &CrateName, num: semver::Version) -> Result<VersionId> {+ let mut state = self.state.lock();+ let id = VersionId(state.next_id());+ let publisher = state.token_user;+ let simulated_crate = state.simulated_crate(name)?;+ simulated_crate.versions.insert(+ 0,+ Version {+ id,+ crate_name: name.clone(),+ num: num.clone(),+ created_at: test_timestamp(),+ updated_at: test_timestamp(),+ downloads: 0,+ yanked: false,+ yank_message: None,+ license: None,+ rust_version: None,+ edition: None,+ crate_size: 0,+ checksum: String::new(),+ published_by: Some(User {+ id: publisher,+ login: format!("user-{}", publisher.0),+ name: None,+ }),+ },+ );+ simulated_crate.krate.default_version = Some(num);+ simulated_crate.krate.num_versions += 1;+ Ok(id)+ }}#[async_trait]@@ -39,4 +166,69 @@ impl CratesIoClient for SimulatedCratesIoClient {async fn get_site_metadata(&self) -> Result<SiteMetadata> {Ok(self.state.lock().site_metadata.clone())}++ async fn list_crates(&self, query: &CrateListQuery) -> Result<Vec<Crate>> {+ let state = self.state.lock();+ let has_value = |values: &Option<Vec<String>>, value: &String| {+ values.as_ref().is_some_and(|values| values.contains(value))+ };+ let mut crates = state+ .crates+ .values()+ .filter(|simulated_crate| {+ let krate = &simulated_crate.krate;+ let matches_owner = match (&query.keyword, query.user_id) {+ (Some(keyword), _) => has_value(&krate.keywords, keyword),+ (None, Some(user_id)) => simulated_crate.owners.contains(&user_id),+ (None, None) => true,+ };+ matches_owner+ && query.search.as_ref().is_none_or(|search| {+ krate+ .name+ .as_str()+ .to_lowercase()+ .contains(&search.to_lowercase())+ })+ && query+ .category+ .as_ref()+ .is_none_or(|category| has_value(&krate.categories, category))+ && !(query.exclude_yanked && krate.yanked)+ })+ .map(|simulated_crate| listed(&simulated_crate.krate))+ .collect::<Vec<_>>();++ // Crates are stored by name, so they are already alphabetical. An exact+ // match stands in for crates.io's relevance order when searching.+ if let Some(search) = &query.search {+ crates.sort_by_key(|krate| krate.name.as_str() != search);+ }+ Ok(crates)+ }++ async fn get_crate(&self, name: &CrateName) -> Result<CrateDetails> {+ let mut state = self.state.lock();+ let simulated_crate = state.simulated_crate(name)?;+ Ok(CrateDetails {+ krate: simulated_crate.krate.clone(),+ versions: simulated_crate.versions.clone(),+ })+ }++ async fn update_crate(&self, name: &CrateName, update: &CrateUpdate) -> Result<Crate> {+ let mut state = self.state.lock();+ let token_user = state.token_user;+ let simulated_crate = state.simulated_crate(name)?;+ if !simulated_crate.owners.contains(&token_user) {+ return Err(CratesIoError::PermissionDenied {+ status: StatusCode::FORBIDDEN,+ message: "only owners have permission to modify crate settings".to_owned(),+ });+ }+ if let Some(trustpub_only) = update.trustpub_only {+ simulated_crate.krate.trustpub_only = trustpub_only;+ }+ Ok(listed(&simulated_crate.krate))+ }}diff --git a/tooling/crates_io_client/src/tests.rs b/tooling/crates_io_client/src/tests.rsindex 365c4d5e982..2ce93d0dcc0 100644--- a/tooling/crates_io_client/src/tests.rs+++ b/tooling/crates_io_client/src/tests.rs@@ -3,8 +3,9 @@ use std::{time::{Duration, Instant},};+use futures::AsyncReadExt as _;use http_client::{- AsyncBody, FakeHttpClient, HttpClient, Response, StatusCode,+ AsyncBody, FakeHttpClient, HttpClient, Method, Response, StatusCode,http::{HeaderMap, header},};use parking_lot::Mutex;@@ -16,7 +17,10 @@ use crate::{};struct RecordedRequest {+ method: Method,+ uri: String,headers: HeaderMap,+ body: String,}/// Returns a fake HTTP client that records each request and answers it with@@ -28,13 +32,19 @@ fn recording_fake_http_client(let respond = Arc::new(respond);let fake_http_client = FakeHttpClient::create({let requests = requests.clone();- move |request| {+ move |mut request| {let requests = requests.clone();let respond = respond.clone();async move {- let (status, response_body) = respond(&request.uri().to_string());+ let mut body = String::new();+ request.body_mut().read_to_string(&mut body).await?;+ let uri = request.uri().to_string();+ let (status, response_body) = respond(&uri);requests.lock().push(RecordedRequest {+ method: request.method().clone(),+ uri,headers: request.headers().clone(),+ body,});Ok(Response::builder().status(status)@@ -45,6 +55,26 @@ fn recording_fake_http_client((fake_http_client, requests)}+fn crate_json(name: &str) -> Value {+ json!({+ "name": name,+ "description": null,+ "homepage": null,+ "documentation": null,+ "repository": null,+ "created_at": "2026-01-01T00:00:00Z",+ "updated_at": "2026-01-01T00:00:00Z",+ "downloads": 1,+ "recent_downloads": 1,+ "default_version": "0.1.0",+ "num_versions": 1,+ "yanked": false,+ "trustpub_only": true,+ "keywords": null,+ "categories": null,+ })+}+#[test]fn test_requests_identify_zed_and_authenticate_with_api_token() {let (fake_http_client, requests) = recording_fake_http_client(|_| {@@ -100,6 +130,74 @@ fn test_error_responses_distinguish_permission_and_upstream_failures() {));}+#[test]+fn test_list_crates_follows_next_page_until_exhausted() {+ let (fake_http_client, requests) = recording_fake_http_client(|uri| {+ let page = if uri.contains("seek=") {+ json!({+ "crates": [crate_json("gpui_util")],+ "meta": { "total": 2, "next_page": null, "prev_page": null },+ })+ } else {+ json!({+ "crates": [crate_json("gpui")],+ "meta": {+ "total": 2,+ "next_page": "?user_id=7&per_page=100&seek=abc",+ "prev_page": null,+ },+ })+ };+ (StatusCode::OK, page)+ });+ let client = RealCratesIoClient::new(fake_http_client, ApiToken::new("secret"));++ let crates = smol::block_on(client.list_crates(&CrateListQuery {+ user_id: Some(UserId(7)),+ exclude_yanked: true,+ ..Default::default()+ }))+ .unwrap();++ let names = crates+ .iter()+ .map(|krate| krate.name.as_str())+ .collect::<Vec<_>>();+ assert_eq!(names, ["gpui", "gpui_util"]);+ let requests = requests.lock();+ assert_eq!(+ requests[0].uri,+ "https://crates.io/api/v1/crates?per_page=100&user_id=7&include_yanked=no"+ );+ assert_eq!(+ requests[1].uri,+ "https://crates.io/api/v1/crates?user_id=7&per_page=100&seek=abc"+ );+}++#[test]+fn test_update_crate_sends_settings_wrapped_in_crate_object() {+ let (fake_http_client, requests) =+ recording_fake_http_client(|_| (StatusCode::OK, json!({ "crate": crate_json("gpui") })));+ let client = RealCratesIoClient::new(fake_http_client, ApiToken::new("secret"));++ smol::block_on(client.update_crate(+ &CrateName::new("gpui"),+ &CrateUpdate {+ trustpub_only: Some(true),+ },+ ))+ .unwrap();++ let requests = requests.lock();+ assert_eq!(requests[0].method, Method::PATCH);+ assert_eq!(requests[0].uri, "https://crates.io/api/v1/crates/gpui");+ assert_eq!(+ serde_json::from_str::<Value>(&requests[0].body).unwrap(),+ json!({ "crate": { "trustpub_only": true } })+ );+}+#[test]fn test_throttle_spaces_requests_by_minimum_interval() {let throttle = RequestThrottle::new();diff --git a/tooling/crates_io_client/src/types.rs b/tooling/crates_io_client/src/types.rsindex 58fda9e2a7c..5272c94660f 100644--- a/tooling/crates_io_client/src/types.rs+++ b/tooling/crates_io_client/src/types.rs@@ -1,6 +1,33 @@+use std::fmt;++use chrono::{DateTime, Utc};use serde::{Deserialize, Serialize};use url::Url;+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]+pub struct UserId(pub i32);+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]+pub struct VersionId(pub i32);++#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]+pub struct CrateName(pub String);++impl CrateName {+ pub fn new(name: impl Into<String>) -> Self {+ Self(name.into())+ }++ pub fn as_str(&self) -> &str {+ &self.0+ }+}++impl fmt::Display for CrateName {+ fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {+ formatter.write_str(&self.0)+ }+}+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]pub struct SiteMetadata {pub deployed_sha: String,@@ -10,3 +37,85 @@ pub struct SiteMetadata {#[serde(default)]pub banner_message: Option<String>,}++/// Filters for listing crates. crates.io ignores `user_id` when `keyword` is+/// set.+#[derive(Debug, Clone, Default, PartialEq, Eq)]+pub struct CrateListQuery {+ /// Results are sorted by relevance with a search and alphabetically+ /// otherwise. Only the first 1000 results of a search can be listed.+ pub search: Option<String>,+ /// Also matches crates in subcategories.+ pub category: Option<String>,+ pub keyword: Option<String>,+ pub user_id: Option<UserId>,+ /// Excludes crates whose versions are all yanked.+ pub exclude_yanked: bool,+}++#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]+pub struct Crate {+ pub name: CrateName,+ pub description: Option<String>,+ pub homepage: Option<String>,+ pub documentation: Option<String>,+ pub repository: Option<String>,+ pub created_at: DateTime<Utc>,+ pub updated_at: DateTime<Utc>,+ pub downloads: u64,+ /// Downloads in the last 90 days.+ pub recent_downloads: Option<u64>,+ /// The version shown on the crate's page. `None` for crates without a+ /// published version.+ pub default_version: Option<semver::Version>,+ pub num_versions: u32,+ /// Whether all versions are yanked.+ pub yanked: bool,+ pub trustpub_only: bool,+ /// Keyword and category IDs, only included when requesting a single crate.+ pub keywords: Option<Vec<String>>,+ pub categories: Option<Vec<String>>,+}++#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]+pub struct Version {+ pub id: VersionId,+ #[serde(rename = "crate")]+ pub crate_name: CrateName,+ pub num: semver::Version,+ pub created_at: DateTime<Utc>,+ pub updated_at: DateTime<Utc>,+ pub downloads: u64,+ pub yanked: bool,+ pub yank_message: Option<String>,+ pub license: Option<String>,+ pub rust_version: Option<String>,+ pub edition: Option<String>,+ /// Size of the compressed crate file in bytes.+ pub crate_size: u64,+ /// Hex-encoded SHA256 of the compressed crate file.+ pub checksum: String,+ /// `None` for versions published before crates.io recorded publishers.+ pub published_by: Option<User>,+}++#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]+pub struct User {+ pub id: UserId,+ pub login: String,+ pub name: Option<String>,+}++#[derive(Debug, Clone, PartialEq, Eq)]+pub struct CrateDetails {+ pub krate: Crate,+ pub versions: Vec<Version>,+}++/// Crate settings to change. Settings left as `None` keep their value.+#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize)]+pub struct CrateUpdate {+ /// Whether the crate can only be published through Trusted Publishing.+ #[serde(skip_serializing_if = "Option::is_none")]+ pub trustpub_only: Option<bool>,+}