Skip to content

armgit / zed

Public Imported from GitHub
0

Code at the speed of thought – Zed is a high-performance, multiplayer code editor from the creators of Atom and Tree-sitter.

crates_io_client: Add endpoints for listing, getting, and updating crates (#66438)

Finn Evers4ebf3eac14d5
diff --git a/Cargo.lock b/Cargo.lock
index ed76ba16512..6695899a1f7 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -4104,10 +4104,12 @@ version = "0.1.0"
dependencies = [
"anyhow",
"async-trait",
+ "chrono",
"futures 0.3.34",
"http_client",
"itertools 0.14.0",
"parking_lot",
+ "semver",
"serde",
"serde_json",
"smol",
diff --git a/tooling/crates_io_client/Cargo.toml b/tooling/crates_io_client/Cargo.toml
index 3928072748f..ea72a581839 100644
--- a/tooling/crates_io_client/Cargo.toml
+++ b/tooling/crates_io_client/Cargo.toml
@@ -19,11 +19,13 @@ doctest = false
[dependencies]
anyhow.workspace = true
async-trait.workspace = true
+chrono.workspace = true
futures.workspace = true
http_client.workspace = true
itertools.workspace = true
parking_lot.workspace = true
serde = { workspace = true, features = ["derive"] }
+semver.workspace = true
serde_json.workspace = true
smol.workspace = true
thiserror.workspace = true
diff --git a/tooling/crates_io_client/src/crates_io_client.rs b/tooling/crates_io_client/src/crates_io_client.rs
index ec158c59054..1b6e627dc44 100644
--- a/tooling/crates_io_client/src/crates_io_client.rs
+++ b/tooling/crates_io_client/src/crates_io_client.rs
@@ -19,6 +19,16 @@ pub type Result<T, E = CratesIoError> = std::result::Result<T, E>;
#[async_trait]
pub trait CratesIoClient: Send + Sync {
async fn get_site_metadata(&self) -> Result<SiteMetadata>;
+
+ /// Lists every crate matching the query, requesting one page of 100 crates
+ /// per second.
+ async fn list_crates(&self, query: &CrateListQuery) -> Result<Vec<Crate>>;
+
+ async fn get_crate(&self, name: &CrateName) -> Result<CrateDetails>;
+
+ /// Requires a token with the `trusted-publishing` endpoint scope that
+ /// belongs to an owner of the crate.
+ async fn update_crate(&self, name: &CrateName, update: &CrateUpdate) -> Result<Crate>;
}
#[derive(Debug, thiserror::Error)]
diff --git a/tooling/crates_io_client/src/real_crates_io_client.rs b/tooling/crates_io_client/src/real_crates_io_client.rs
index afcc22f78a1..f282a3f9c93 100644
--- a/tooling/crates_io_client/src/real_crates_io_client.rs
+++ b/tooling/crates_io_client/src/real_crates_io_client.rs
@@ -8,7 +8,7 @@ use async_trait::async_trait;
use futures::AsyncReadExt as _;
use http_client::{AsyncBody, HttpClient, Method, Request, Response, http::header};
use parking_lot::Mutex;
-use serde::de::DeserializeOwned;
+use serde::{Deserialize, Serialize, de::DeserializeOwned};
use url::Url;
use crate::*;
@@ -17,6 +17,7 @@ const BASE_URL: &str = "https://crates.io/api/v1";
// User agent and request rate requirements: https://crates.io/data-access
const USER_AGENT: &str = "Zed GPUI releases (https://github.com/zed-industries/zed)";
pub(crate) const MIN_REQUEST_INTERVAL: Duration = Duration::from_secs(1);
+const MAX_PER_PAGE: u32 = 100;
/// Reserving the send time before waiting makes concurrent requests queue up
/// behind each other instead of all waking at the same instant.
@@ -59,6 +60,14 @@ impl RealCratesIoClient {
Ok(Url::parse(&format!("{BASE_URL}{path}")).context("Invalid crates.io URL")?)
}
+ fn crate_url(name: &CrateName) -> Result<Url> {
+ let mut url = Self::url("/crates")?;
+ url.path_segments_mut()
+ .map_err(|()| anyhow::anyhow!("crates.io URL cannot have path segments"))?
+ .push(name.as_str());
+ Ok(url)
+ }
+
async fn request_response(
&self,
method: Method,
@@ -124,4 +133,92 @@ impl CratesIoClient for RealCratesIoClient {
self.request_json(Method::GET, Self::url("/site_metadata")?, None)
.await
}
+
+ async fn list_crates(&self, query: &CrateListQuery) -> Result<Vec<Crate>> {
+ #[derive(Deserialize)]
+ struct CrateListResponse {
+ crates: Vec<Crate>,
+ meta: CrateListMeta,
+ }
+
+ #[derive(Deserialize)]
+ struct CrateListMeta {
+ /// A query string for the next page, which preserves the filters.
+ next_page: Option<String>,
+ }
+
+ let mut url = Self::url("/crates")?;
+ {
+ let mut pairs = url.query_pairs_mut();
+ pairs.append_pair("per_page", &MAX_PER_PAGE.to_string());
+ if let Some(search) = &query.search {
+ pairs.append_pair("q", search);
+ }
+ if let Some(category) = &query.category {
+ pairs.append_pair("category", category);
+ }
+ if let Some(keyword) = &query.keyword {
+ pairs.append_pair("keyword", keyword);
+ }
+ if let Some(user_id) = query.user_id {
+ pairs.append_pair("user_id", &user_id.0.to_string());
+ }
+ if query.exclude_yanked {
+ pairs.append_pair("include_yanked", "no");
+ }
+ }
+
+ let mut crates = Vec::new();
+ loop {
+ let mut page: CrateListResponse = self.request_json(Method::GET, url, None).await?;
+ let is_last_page = page.crates.is_empty();
+ crates.append(&mut page.crates);
+
+ match page.meta.next_page {
+ Some(next_page) if !is_last_page => {
+ url = Self::url(&format!("/crates{next_page}"))?;
+ }
+ _ => return Ok(crates),
+ }
+ }
+ }
+
+ async fn get_crate(&self, name: &CrateName) -> Result<CrateDetails> {
+ #[derive(Deserialize)]
+ struct CrateResponse {
+ #[serde(rename = "crate")]
+ krate: Crate,
+ versions: Option<Vec<Version>>,
+ }
+
+ let mut url = Self::crate_url(name)?;
+ url.query_pairs_mut()
+ .append_pair("include", "versions,keywords,categories");
+ let response: CrateResponse = self.request_json(Method::GET, url, None).await?;
+ Ok(CrateDetails {
+ krate: response.krate,
+ versions: response.versions.unwrap_or_default(),
+ })
+ }
+
+ async fn update_crate(&self, name: &CrateName, update: &CrateUpdate) -> Result<Crate> {
+ #[derive(Serialize)]
+ struct UpdateRequest<'a> {
+ #[serde(rename = "crate")]
+ krate: &'a CrateUpdate,
+ }
+
+ #[derive(Deserialize)]
+ struct UpdateResponse {
+ #[serde(rename = "crate")]
+ krate: Crate,
+ }
+
+ let body = serde_json::to_vec(&UpdateRequest { krate: update })
+ .context("Failed to serialize crate update")?;
+ let response: UpdateResponse = self
+ .request_json(Method::PATCH, Self::crate_url(name)?, Some(body))
+ .await?;
+ Ok(response.krate)
+ }
}
diff --git a/tooling/crates_io_client/src/simulated_crates_io_client.rs b/tooling/crates_io_client/src/simulated_crates_io_client.rs
index 9268455b0ea..0f0afd3ab6e 100644
--- a/tooling/crates_io_client/src/simulated_crates_io_client.rs
+++ b/tooling/crates_io_client/src/simulated_crates_io_client.rs
@@ -1,6 +1,8 @@
-use std::sync::Arc;
+use std::{collections::BTreeMap, sync::Arc};
use async_trait::async_trait;
+use chrono::{DateTime, Utc};
+use http_client::StatusCode;
use parking_lot::Mutex;
use url::Url;
@@ -12,9 +14,47 @@ pub struct SimulatedCratesIoClient {
struct CratesIoState {
site_metadata: SiteMetadata,
+ token_user: UserId,
+ crates: BTreeMap<CrateName, SimulatedCrate>,
+ latest_id: i32,
+}
+
+struct SimulatedCrate {
+ krate: Crate,
+ owners: Vec<UserId>,
+ versions: Vec<Version>,
+}
+
+impl CratesIoState {
+ fn next_id(&mut self) -> i32 {
+ self.latest_id += 1;
+ self.latest_id
+ }
+
+ fn simulated_crate(&mut self, name: &CrateName) -> Result<&mut SimulatedCrate> {
+ self.crates
+ .get_mut(name)
+ .ok_or_else(|| CratesIoError::NotFound {
+ message: format!("crate `{name}` does not exist"),
+ })
+ }
+}
+
+/// Keyword and category IDs are only part of single-crate responses.
+fn listed(krate: &Crate) -> Crate {
+ Crate {
+ keywords: None,
+ categories: None,
+ ..krate.clone()
+ }
+}
+
+fn test_timestamp() -> DateTime<Utc> {
+ DateTime::from_timestamp(1_767_225_600, 0).unwrap_or_default()
}
impl SimulatedCratesIoClient {
+ /// Creates a registry whose API token belongs to [`Self::token_user`].
pub fn new() -> Arc<Self> {
Arc::new(Self {
state: Mutex::new(CratesIoState {
@@ -25,13 +65,100 @@ impl SimulatedCratesIoClient {
read_only: false,
banner_message: None,
},
+ token_user: UserId(1),
+ crates: BTreeMap::new(),
+ latest_id: 1,
}),
})
}
+ pub fn token_user(&self) -> UserId {
+ self.state.lock().token_user
+ }
+
+ pub fn test_user(&self) -> UserId {
+ UserId(self.state.lock().next_id())
+ }
+
pub fn test_site_metadata(&self, site_metadata: SiteMetadata) {
self.state.lock().site_metadata = site_metadata;
}
+
+ /// Adds a crate without versions, replacing any crate with the same name.
+ pub fn test_crate(&self, name: &str, owner: UserId) -> CrateName {
+ let name = CrateName::new(name);
+ self.state.lock().crates.insert(
+ name.clone(),
+ SimulatedCrate {
+ krate: Crate {
+ name: name.clone(),
+ description: None,
+ homepage: None,
+ documentation: None,
+ repository: None,
+ created_at: test_timestamp(),
+ updated_at: test_timestamp(),
+ downloads: 0,
+ recent_downloads: Some(0),
+ default_version: None,
+ num_versions: 0,
+ yanked: false,
+ trustpub_only: false,
+ keywords: Some(Vec::new()),
+ categories: Some(Vec::new()),
+ },
+ owners: vec![owner],
+ versions: Vec::new(),
+ },
+ );
+ name
+ }
+
+ /// Changes crate fields that have no dedicated helper, such as its
+ /// keywords or download counts.
+ pub fn update_test_crate(
+ &self,
+ name: &CrateName,
+ update: impl FnOnce(&mut Crate),
+ ) -> Result<()> {
+ let mut state = self.state.lock();
+ update(&mut state.simulated_crate(name)?.krate);
+ Ok(())
+ }
+
+ /// Publishes a version, which becomes the crate's default version.
+ pub fn test_version(&self, name: &CrateName, num: semver::Version) -> Result<VersionId> {
+ let mut state = self.state.lock();
+ let id = VersionId(state.next_id());
+ let publisher = state.token_user;
+ let simulated_crate = state.simulated_crate(name)?;
+ simulated_crate.versions.insert(
+ 0,
+ Version {
+ id,
+ crate_name: name.clone(),
+ num: num.clone(),
+ created_at: test_timestamp(),
+ updated_at: test_timestamp(),
+ downloads: 0,
+ yanked: false,
+ yank_message: None,
+ license: None,
+ rust_version: None,
+ edition: None,
+ crate_size: 0,
+ checksum: String::new(),
+ published_by: Some(User {
+ id: publisher,
+ login: format!("user-{}", publisher.0),
+ name: None,
+ }),
+ },
+ );
+ simulated_crate.krate.default_version = Some(num);
+ simulated_crate.krate.num_versions += 1;
+ Ok(id)
+ }
}
#[async_trait]
@@ -39,4 +166,69 @@ impl CratesIoClient for SimulatedCratesIoClient {
async fn get_site_metadata(&self) -> Result<SiteMetadata> {
Ok(self.state.lock().site_metadata.clone())
}
+
+ async fn list_crates(&self, query: &CrateListQuery) -> Result<Vec<Crate>> {
+ let state = self.state.lock();
+ let has_value = |values: &Option<Vec<String>>, value: &String| {
+ values.as_ref().is_some_and(|values| values.contains(value))
+ };
+ let mut crates = state
+ .crates
+ .values()
+ .filter(|simulated_crate| {
+ let krate = &simulated_crate.krate;
+ let matches_owner = match (&query.keyword, query.user_id) {
+ (Some(keyword), _) => has_value(&krate.keywords, keyword),
+ (None, Some(user_id)) => simulated_crate.owners.contains(&user_id),
+ (None, None) => true,
+ };
+ matches_owner
+ && query.search.as_ref().is_none_or(|search| {
+ krate
+ .name
+ .as_str()
+ .to_lowercase()
+ .contains(&search.to_lowercase())
+ })
+ && query
+ .category
+ .as_ref()
+ .is_none_or(|category| has_value(&krate.categories, category))
+ && !(query.exclude_yanked && krate.yanked)
+ })
+ .map(|simulated_crate| listed(&simulated_crate.krate))
+ .collect::<Vec<_>>();
+
+ // Crates are stored by name, so they are already alphabetical. An exact
+ // match stands in for crates.io's relevance order when searching.
+ if let Some(search) = &query.search {
+ crates.sort_by_key(|krate| krate.name.as_str() != search);
+ }
+ Ok(crates)
+ }
+
+ async fn get_crate(&self, name: &CrateName) -> Result<CrateDetails> {
+ let mut state = self.state.lock();
+ let simulated_crate = state.simulated_crate(name)?;
+ Ok(CrateDetails {
+ krate: simulated_crate.krate.clone(),
+ versions: simulated_crate.versions.clone(),
+ })
+ }
+
+ async fn update_crate(&self, name: &CrateName, update: &CrateUpdate) -> Result<Crate> {
+ let mut state = self.state.lock();
+ let token_user = state.token_user;
+ let simulated_crate = state.simulated_crate(name)?;
+ if !simulated_crate.owners.contains(&token_user) {
+ return Err(CratesIoError::PermissionDenied {
+ status: StatusCode::FORBIDDEN,
+ message: "only owners have permission to modify crate settings".to_owned(),
+ });
+ }
+ if let Some(trustpub_only) = update.trustpub_only {
+ simulated_crate.krate.trustpub_only = trustpub_only;
+ }
+ Ok(listed(&simulated_crate.krate))
+ }
}
diff --git a/tooling/crates_io_client/src/tests.rs b/tooling/crates_io_client/src/tests.rs
index 365c4d5e982..2ce93d0dcc0 100644
--- a/tooling/crates_io_client/src/tests.rs
+++ b/tooling/crates_io_client/src/tests.rs
@@ -3,8 +3,9 @@ use std::{
time::{Duration, Instant},
};
+use futures::AsyncReadExt as _;
use http_client::{
- AsyncBody, FakeHttpClient, HttpClient, Response, StatusCode,
+ AsyncBody, FakeHttpClient, HttpClient, Method, Response, StatusCode,
http::{HeaderMap, header},
};
use parking_lot::Mutex;
@@ -16,7 +17,10 @@ use crate::{
};
struct RecordedRequest {
+ method: Method,
+ uri: String,
headers: HeaderMap,
+ body: String,
}
/// Returns a fake HTTP client that records each request and answers it with
@@ -28,13 +32,19 @@ fn recording_fake_http_client(
let respond = Arc::new(respond);
let fake_http_client = FakeHttpClient::create({
let requests = requests.clone();
- move |request| {
+ move |mut request| {
let requests = requests.clone();
let respond = respond.clone();
async move {
- let (status, response_body) = respond(&request.uri().to_string());
+ let mut body = String::new();
+ request.body_mut().read_to_string(&mut body).await?;
+ let uri = request.uri().to_string();
+ let (status, response_body) = respond(&uri);
requests.lock().push(RecordedRequest {
+ method: request.method().clone(),
+ uri,
headers: request.headers().clone(),
+ body,
});
Ok(Response::builder()
.status(status)
@@ -45,6 +55,26 @@ fn recording_fake_http_client(
(fake_http_client, requests)
}
+fn crate_json(name: &str) -> Value {
+ json!({
+ "name": name,
+ "description": null,
+ "homepage": null,
+ "documentation": null,
+ "repository": null,
+ "created_at": "2026-01-01T00:00:00Z",
+ "updated_at": "2026-01-01T00:00:00Z",
+ "downloads": 1,
+ "recent_downloads": 1,
+ "default_version": "0.1.0",
+ "num_versions": 1,
+ "yanked": false,
+ "trustpub_only": true,
+ "keywords": null,
+ "categories": null,
+ })
+}
+
#[test]
fn test_requests_identify_zed_and_authenticate_with_api_token() {
let (fake_http_client, requests) = recording_fake_http_client(|_| {
@@ -100,6 +130,74 @@ fn test_error_responses_distinguish_permission_and_upstream_failures() {
));
}
+#[test]
+fn test_list_crates_follows_next_page_until_exhausted() {
+ let (fake_http_client, requests) = recording_fake_http_client(|uri| {
+ let page = if uri.contains("seek=") {
+ json!({
+ "crates": [crate_json("gpui_util")],
+ "meta": { "total": 2, "next_page": null, "prev_page": null },
+ })
+ } else {
+ json!({
+ "crates": [crate_json("gpui")],
+ "meta": {
+ "total": 2,
+ "next_page": "?user_id=7&per_page=100&seek=abc",
+ "prev_page": null,
+ },
+ })
+ };
+ (StatusCode::OK, page)
+ });
+ let client = RealCratesIoClient::new(fake_http_client, ApiToken::new("secret"));
+
+ let crates = smol::block_on(client.list_crates(&CrateListQuery {
+ user_id: Some(UserId(7)),
+ exclude_yanked: true,
+ ..Default::default()
+ }))
+ .unwrap();
+
+ let names = crates
+ .iter()
+ .map(|krate| krate.name.as_str())
+ .collect::<Vec<_>>();
+ assert_eq!(names, ["gpui", "gpui_util"]);
+ let requests = requests.lock();
+ assert_eq!(
+ requests[0].uri,
+ "https://crates.io/api/v1/crates?per_page=100&user_id=7&include_yanked=no"
+ );
+ assert_eq!(
+ requests[1].uri,
+ "https://crates.io/api/v1/crates?user_id=7&per_page=100&seek=abc"
+ );
+}
+
+#[test]
+fn test_update_crate_sends_settings_wrapped_in_crate_object() {
+ let (fake_http_client, requests) =
+ recording_fake_http_client(|_| (StatusCode::OK, json!({ "crate": crate_json("gpui") })));
+ let client = RealCratesIoClient::new(fake_http_client, ApiToken::new("secret"));
+
+ smol::block_on(client.update_crate(
+ &CrateName::new("gpui"),
+ &CrateUpdate {
+ trustpub_only: Some(true),
+ },
+ ))
+ .unwrap();
+
+ let requests = requests.lock();
+ assert_eq!(requests[0].method, Method::PATCH);
+ assert_eq!(requests[0].uri, "https://crates.io/api/v1/crates/gpui");
+ assert_eq!(
+ serde_json::from_str::<Value>(&requests[0].body).unwrap(),
+ json!({ "crate": { "trustpub_only": true } })
+ );
+}
+
#[test]
fn test_throttle_spaces_requests_by_minimum_interval() {
let throttle = RequestThrottle::new();
diff --git a/tooling/crates_io_client/src/types.rs b/tooling/crates_io_client/src/types.rs
index 58fda9e2a7c..5272c94660f 100644
--- a/tooling/crates_io_client/src/types.rs
+++ b/tooling/crates_io_client/src/types.rs
@@ -1,6 +1,33 @@
+use std::fmt;
+
+use chrono::{DateTime, Utc};
use serde::{Deserialize, Serialize};
use url::Url;
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
+pub struct UserId(pub i32);
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
+pub struct VersionId(pub i32);
+
+#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
+pub struct CrateName(pub String);
+
+impl CrateName {
+ pub fn new(name: impl Into<String>) -> Self {
+ Self(name.into())
+ }
+
+ pub fn as_str(&self) -> &str {
+ &self.0
+ }
+}
+
+impl fmt::Display for CrateName {
+ fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
+ formatter.write_str(&self.0)
+ }
+}
+
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct SiteMetadata {
pub deployed_sha: String,
@@ -10,3 +37,85 @@ pub struct SiteMetadata {
#[serde(default)]
pub banner_message: Option<String>,
}
+
+/// Filters for listing crates. crates.io ignores `user_id` when `keyword` is
+/// set.
+#[derive(Debug, Clone, Default, PartialEq, Eq)]
+pub struct CrateListQuery {
+ /// Results are sorted by relevance with a search and alphabetically
+ /// otherwise. Only the first 1000 results of a search can be listed.
+ pub search: Option<String>,
+ /// Also matches crates in subcategories.
+ pub category: Option<String>,
+ pub keyword: Option<String>,
+ pub user_id: Option<UserId>,
+ /// Excludes crates whose versions are all yanked.
+ pub exclude_yanked: bool,
+}
+
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+pub struct Crate {
+ pub name: CrateName,
+ pub description: Option<String>,
+ pub homepage: Option<String>,
+ pub documentation: Option<String>,
+ pub repository: Option<String>,
+ pub created_at: DateTime<Utc>,
+ pub updated_at: DateTime<Utc>,
+ pub downloads: u64,
+ /// Downloads in the last 90 days.
+ pub recent_downloads: Option<u64>,
+ /// The version shown on the crate's page. `None` for crates without a
+ /// published version.
+ pub default_version: Option<semver::Version>,
+ pub num_versions: u32,
+ /// Whether all versions are yanked.
+ pub yanked: bool,
+ pub trustpub_only: bool,
+ /// Keyword and category IDs, only included when requesting a single crate.
+ pub keywords: Option<Vec<String>>,
+ pub categories: Option<Vec<String>>,
+}
+
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+pub struct Version {
+ pub id: VersionId,
+ #[serde(rename = "crate")]
+ pub crate_name: CrateName,
+ pub num: semver::Version,
+ pub created_at: DateTime<Utc>,
+ pub updated_at: DateTime<Utc>,
+ pub downloads: u64,
+ pub yanked: bool,
+ pub yank_message: Option<String>,
+ pub license: Option<String>,
+ pub rust_version: Option<String>,
+ pub edition: Option<String>,
+ /// Size of the compressed crate file in bytes.
+ pub crate_size: u64,
+ /// Hex-encoded SHA256 of the compressed crate file.
+ pub checksum: String,
+ /// `None` for versions published before crates.io recorded publishers.
+ pub published_by: Option<User>,
+}
+
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+pub struct User {
+ pub id: UserId,
+ pub login: String,
+ pub name: Option<String>,
+}
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct CrateDetails {
+ pub krate: Crate,
+ pub versions: Vec<Version>,
+}
+
+/// Crate settings to change. Settings left as `None` keep their value.
+#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize)]
+pub struct CrateUpdate {
+ /// Whether the crate can only be published through Trusted Publishing.
+ #[serde(skip_serializing_if = "Option::is_none")]
+ pub trustpub_only: Option<bool>,
+}