Skip to content

The IDE you know, rebuilt in Rust. IntelliJ IDEA–style keymap, tool windows and search-everywhere in a fast native app: editor, code insight, debugger, Git, terminal, database and HTTP clients for 60+ languages.

fix(tools): install GitHub releases when the API is rate-limited (#1715)

l0ng-aic1aa092d7505
diff --git a/crates/ide-tools/src/lib.rs b/crates/ide-tools/src/lib.rs
index d0a8b65d5..dbdddfa5b 100644
--- a/crates/ide-tools/src/lib.rs
+++ b/crates/ide-tools/src/lib.rs
@@ -1221,10 +1221,26 @@ impl Tools {
cancel: &CancellationToken,
) -> IdeResult<sources::Download> {
match source {
- ReleaseSource::GitHub { repo } => {
- let json = http.get_text(&sources::github_latest_url(repo), cancel)?;
- sources::github_pick(&json, assets, self.platform)
- }
+ ReleaseSource::GitHub { repo } => match http.get_text(&sources::github_latest_url(repo), cancel) {
+ Ok(json) => sources::github_pick(&json, assets, self.platform),
+ Err(e) if e.kind == ErrorKind::Cancelled => Err(e),
+ // Without a token the API allows 60 requests an hour per IP address, often used
+ // up behind a shared one (403/429); the release's web pages have no such limit.
+ Err(api) => self.resolve_github_web(http, repo, assets, cancel).map_err(|web| {
+ if web.kind == ErrorKind::Cancelled {
+ return web;
+ }
+ tracing::warn!("GitHub release lookup for {repo} failed: {api}; web pages: {web}");
+ IdeError::new(
+ api.kind,
+ format!(
+ "{}\nDownload it from {} and set its Path in Settings › Languages & Frameworks › Language Servers.",
+ api.message,
+ sources::github_latest_page(repo)
+ ),
+ )
+ }),
+ },
ReleaseSource::HashiCorp { product } => {
let json = http.get_text(&sources::hashicorp_latest_url(product), cancel)?;
sources::hashicorp_pick(&json, assets, self.platform)
@@ -1251,6 +1267,27 @@ impl Tools {
}
}
+ /// [`resolve_release`](Self::resolve_release) for a GitHub release, from its web pages.
+ fn resolve_github_web(
+ &self,
+ http: &Http,
+ repo: &str,
+ assets: &[platform::AssetRule],
+ cancel: &CancellationToken,
+ ) -> IdeResult<sources::Download> {
+ let page = http.effective_url(&sources::github_latest_page(repo), cancel)?;
+ let tag = sources::github_tag_from_url(&page)
+ .ok_or_else(|| IdeError::unavailable(format!("{repo} has no published release")))?;
+ let list = http.get_text(&sources::github_assets_page(repo, &tag), cancel)?;
+ match sources::github_pick_web(repo, &tag, &list, None, assets, self.platform) {
+ Err(e) if e.kind == ErrorKind::Unavailable => {
+ let notes = http.get_text(&sources::github_tag_page(repo, &tag), cancel)?;
+ sources::github_pick_web(repo, &tag, &list, Some(&notes), assets, self.platform)
+ }
+ result => result,
+ }
+ }
+
fn install_release(
&self,
spec: &ToolSpec,
diff --git a/crates/ide-tools/src/process.rs b/crates/ide-tools/src/process.rs
index ac8a48b26..7d34423e4 100644
--- a/crates/ide-tools/src/process.rs
+++ b/crates/ide-tools/src/process.rs
@@ -263,6 +263,15 @@ fn curl_takes_revoke_best_effort(version: &str) -> bool {
(major, minor) >= (7, 70) && first.split_whitespace().any(|w| w.eq_ignore_ascii_case("schannel"))
}
+#[derive(Clone, Copy, PartialEq, Eq)]
+enum Request {
+ Get,
+ /// Adoptium uses 404 as well as an empty array for absent platform builds.
+ GetOrEmptyOn404,
+ /// Headers, then the URL redirects ended at.
+ Head,
+}
+
/// HTTP via the system `curl` (shipped with macOS, Windows 10+ and every
/// mainstream Linux), so the IDE carries no TLS stack of its own.
pub struct Http {
@@ -336,26 +345,39 @@ impl Http {
self.get_text_limited(url, None, cancel)
}
+ /// Where `url` ends up after redirects (a HEAD request).
+ pub fn effective_url(&self, url: &str, cancel: &CancellationToken) -> IdeResult<String> {
+ let out = self.get_text_response(url, None, Request::Head, cancel)?;
+ Ok(out.trim_end().rsplit('\n').next().unwrap_or_default().trim().to_owned())
+ }
+
/// GETs `url` as text, failing when the response is larger than `max_bytes`.
pub fn get_text_limited(&self, url: &str, max_bytes: Option<u64>, cancel: &CancellationToken) -> IdeResult<String> {
- self.get_text_response(url, max_bytes, false, cancel)
+ self.get_text_response(url, max_bytes, Request::Get, cancel)
}
/// Adoptium uses 404 as well as an empty array for absent platform builds.
pub fn get_text_or_empty_on_404(&self, url: &str, cancel: &CancellationToken) -> IdeResult<String> {
- self.get_text_response(url, None, true, cancel)
+ self.get_text_response(url, None, Request::GetOrEmptyOn404, cancel)
}
fn get_text_response(
&self,
url: &str,
max_bytes: Option<u64>,
- empty_on_404: bool,
+ request: Request,
cancel: &CancellationToken,
) -> IdeResult<String> {
let (mut cmd, token) = self.base(url, cancel)?;
- if empty_on_404 {
- cmd.args(["--write-out", "\n%{http_code}"]);
+ let empty_on_404 = request == Request::GetOrEmptyOn404;
+ match request {
+ Request::Get => {}
+ Request::GetOrEmptyOn404 => {
+ cmd.args(["--write-out", "\n%{http_code}"]);
+ }
+ Request::Head => {
+ cmd.args(["--head", "--write-out", "\n%{url_effective}"]);
+ }
}
if let Some(max) = max_bytes {
cmd.arg("--max-filesize").arg(max.to_string());
diff --git a/crates/ide-tools/src/sources.rs b/crates/ide-tools/src/sources.rs
index 2337bc75c..0e5da7481 100644
--- a/crates/ide-tools/src/sources.rs
+++ b/crates/ide-tools/src/sources.rs
@@ -47,38 +47,146 @@ pub fn github_latest_url(repo: &str) -> String {
format!("https://api.github.com/repos/{repo}/releases/latest")
}
+/// The latest release's web page; it redirects to `…/releases/tag/<tag>`. Web pages don't
+/// count against the API's rate limit for unauthenticated clients (60 requests an hour).
+pub fn github_latest_page(repo: &str) -> String {
+ format!("https://github.com/{repo}/releases/latest")
+}
+
+/// The tag `github_latest_page` redirected to.
+pub fn github_tag_from_url(url: &str) -> Option<String> {
+ let (_, tag) = url.split_once("/releases/tag/")?;
+ let tag = tag.split(['?', '#']).next().unwrap_or(tag).trim_end_matches('/');
+ (!tag.is_empty()).then(|| percent_decode(tag))
+}
+
+pub fn github_tag_page(repo: &str, tag: &str) -> String {
+ format!("https://github.com/{repo}/releases/tag/{tag}")
+}
+
+/// The asset list the release page loads (HTML).
+pub fn github_assets_page(repo: &str, tag: &str) -> String {
+ format!("https://github.com/{repo}/releases/expanded_assets/{tag}")
+}
+
+/// One file of a release, from the API or its web page.
+struct ReleaseAsset {
+ name: String,
+ url: String,
+ size: Option<u64>,
+ sha256: Option<String>,
+}
+
/// Picks the platform asset from a GitHub `releases/latest` response. Links
/// in the release notes count as assets too (some projects host binaries on
/// a CDN and only link them).
pub fn github_pick(json: &str, rules: &[AssetRule], platform: Platform) -> IdeResult<Download> {
let rel: GhRelease = serde_json::from_str(json).map_err(|e| proto(format!("unexpected GitHub response: {e}")))?;
- let version = rel.tag_name.trim_start_matches('v').to_string();
- let names: Vec<&str> = rel.assets.iter().map(|a| a.name.as_str()).collect();
+ let assets: Vec<ReleaseAsset> = rel
+ .assets
+ .into_iter()
+ .map(|a| ReleaseAsset {
+ sha256: a.digest.as_deref().and_then(|d| d.strip_prefix("sha256:")).map(str::to_ascii_lowercase),
+ name: a.name,
+ url: a.browser_download_url,
+ size: a.size,
+ })
+ .collect();
+ pick_release(&rel.tag_name, &assets, rel.body.as_deref().unwrap_or(""), rules, platform)
+}
+
+/// [`github_pick`] from the release's web pages: `assets_html` from [`github_assets_page`] and,
+/// for projects that only link their builds, `notes_html` from [`github_tag_page`].
+pub fn github_pick_web(
+ repo: &str,
+ tag: &str,
+ assets_html: &str,
+ notes_html: Option<&str>,
+ rules: &[AssetRule],
+ platform: Platform,
+) -> IdeResult<Download> {
+ let prefix = format!("href=\"/{repo}/releases/download/");
+ let mut assets: Vec<ReleaseAsset> = Vec::new();
+ let mut rest = assets_html;
+ while let Some(i) = rest.find(&prefix) {
+ let tail = &rest[i + "href=\"".len()..];
+ let Some(end) = tail.find('"') else { break };
+ let path = html_unescape(&tail[..end]);
+ rest = &tail[end..];
+ let name = percent_decode(file_name(&path));
+ if name.is_empty() || assets.iter().any(|a| a.name == name) {
+ continue;
+ }
+ // `<clipboard-copy aria-label="Copy to clipboard digest for <name>" … value="sha256:<hex>">`
+ let sha256 = assets_html.find(&format!("digest for {}\"", html_escape(&name))).and_then(|at| {
+ let tag_end = assets_html[at..].find('>').map_or(assets_html.len(), |e| at + e);
+ let value = assets_html[at..tag_end].split_once("value=\"sha256:")?.1;
+ let hex: String = value.chars().take_while(char::is_ascii_hexdigit).collect();
+ (hex.len() == 64).then(|| hex.to_ascii_lowercase())
+ });
+ assets.push(ReleaseAsset { url: format!("https://github.com{path}"), name, size: None, sha256 });
+ }
+ pick_release(tag, &assets, &html_unescape(notes_html.unwrap_or("")), rules, platform)
+}
+
+fn pick_release(
+ tag: &str,
+ assets: &[ReleaseAsset],
+ notes: &str,
+ rules: &[AssetRule],
+ platform: Platform,
+) -> IdeResult<Download> {
+ let version = tag.trim_start_matches('v').to_string();
+ let names: Vec<&str> = assets.iter().map(|a| a.name.as_str()).collect();
if let Some(name) = select_asset(&names, rules, platform) {
- let a = rel.assets.iter().find(|a| a.name == name).expect("selected from list");
- let sha256 = a.digest.as_deref().and_then(|d| d.strip_prefix("sha256:")).map(str::to_ascii_lowercase);
- let checksum_url = rel
- .assets
+ let a = assets.iter().find(|a| a.name == name).expect("selected from list");
+ let checksum_url = assets
.iter()
.find(|c| c.name == format!("{name}.sha256") || c.name == format!("{name}.sha256sum"))
- .map(|c| c.browser_download_url.clone());
+ .map(|c| c.url.clone());
return Ok(Download {
version,
name: a.name.clone(),
- url: a.browser_download_url.clone(),
+ url: a.url.clone(),
size: a.size,
- sha256,
+ sha256: a.sha256.clone(),
checksum_url,
});
}
- let links = extract_links(rel.body.as_deref().unwrap_or(""));
+ let links = extract_links(notes);
let link_names: Vec<&str> = links.iter().map(|u| file_name(u)).collect();
if let Some(name) = select_asset(&link_names, rules, platform) {
let url = links.iter().find(|u| file_name(u) == name).expect("selected from list").clone();
let checksum_url = links.iter().find(|u| *u == &format!("{url}.sha256")).cloned();
return Ok(Download { version, name: name.to_string(), url, size: None, sha256: None, checksum_url });
}
- Err(IdeError::unavailable(format!("release {} has no build for this platform", rel.tag_name)))
+ Err(IdeError::unavailable(format!("release {tag} has no build for this platform")))
+}
+
+fn html_unescape(s: &str) -> String {
+ s.replace("&quot;", "\"").replace("&#39;", "'").replace("&lt;", "<").replace("&gt;", ">").replace("&amp;", "&")
+}
+
+fn html_escape(s: &str) -> String {
+ s.replace('&', "&amp;").replace('"', "&quot;").replace('<', "&lt;").replace('>', "&gt;")
+}
+
+fn percent_decode(s: &str) -> String {
+ let bytes = s.as_bytes();
+ let mut out = Vec::with_capacity(bytes.len());
+ let mut i = 0;
+ while i < bytes.len() {
+ if bytes[i] == b'%'
+ && let Some(b) = s.get(i + 1..i + 3).and_then(|h| u8::from_str_radix(h, 16).ok())
+ {
+ out.push(b);
+ i += 3;
+ } else {
+ out.push(bytes[i]);
+ i += 1;
+ }
+ }
+ String::from_utf8(out).unwrap_or_else(|_| s.to_owned())
}
fn file_name(url: &str) -> &str {
@@ -336,6 +444,71 @@ mod tests {
assert_eq!(d.name, "kotlin-lsp-0.253.10629-linux-x64.zip");
}
+ #[test]
+ fn github_tag_from_redirect() {
+ let url = "https://github.com/Kotlin/kotlin-lsp/releases/tag/kotlin-lsp/v263.7084.0";
+ assert_eq!(github_tag_from_url(url).as_deref(), Some("kotlin-lsp/v263.7084.0"));
+ let url = "https://github.com/o/r/releases/tag/v1.0%2Bbuild";
+ assert_eq!(github_tag_from_url(url).as_deref(), Some("v1.0+build"));
+ // A repository without releases stays on the list.
+ assert_eq!(github_tag_from_url("https://github.com/o/r/releases"), None);
+ }
+
+ /// Trimmed from `github.com/<repo>/releases/expanded_assets/<tag>`.
+ const ASSETS_HTML: &str = r#"<li><div><a href="/LuaLS/lua-language-server/releases/download/3.13.5/lua-language-server-3.13.5-darwin-arm64.tar.gz" rel="nofollow" data-turbo="false" class="wb-break-all">
+ <span class="text-bold">lua-language-server-3.13.5-darwin-arm64.tar.gz</span></a></div>
+ <span class="Truncate-text">sha256:ABCDEF0123456789abcdef0123456789abcdef0123456789abcdef0123456789</span>
+ <clipboard-copy id="clipboard-button-sha256:abc" aria-label="Copy to clipboard digest for lua-language-server-3.13.5-darwin-arm64.tar.gz" type="button" value="sha256:ABCDEF0123456789abcdef0123456789abcdef0123456789abcdef0123456789" class="Button"></clipboard-copy></li>
+ <li><a href="/LuaLS/lua-language-server/releases/download/3.13.5/lua-language-server-3.13.5-linux-x64.tar.gz" rel="nofollow">lua-language-server-3.13.5-linux-x64.tar.gz</a></li>
+ <li><a href="/LuaLS/lua-language-server/archive/refs/tags/3.13.5.zip" rel="nofollow">Source code (zip)</a></li>"#;
+
+ #[test]
+ fn github_web_assets_with_digest() {
+ let rules = find("lua-language-server").unwrap().asset_rules();
+ let d = github_pick_web("LuaLS/lua-language-server", "3.13.5", ASSETS_HTML, None, rules, MAC_ARM).unwrap();
+ assert_eq!(d.version, "3.13.5");
+ assert_eq!(d.name, "lua-language-server-3.13.5-darwin-arm64.tar.gz");
+ assert_eq!(
+ d.url,
+ "https://github.com/LuaLS/lua-language-server/releases/download/3.13.5/lua-language-server-3.13.5-darwin-arm64.tar.gz"
+ );
+ assert_eq!(d.sha256.as_deref(), Some("abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789"));
+ let d = github_pick_web("LuaLS/lua-language-server", "3.13.5", ASSETS_HTML, None, rules, LINUX).unwrap();
+ assert_eq!(d.sha256, None);
+ let win = Platform { os: Os::Windows, arch: Arch::X64 };
+ let err = github_pick_web("LuaLS/lua-language-server", "3.13.5", ASSETS_HTML, None, rules, win).unwrap_err();
+ assert_eq!(err.kind, ErrorKind::Unavailable);
+ }
+
+ #[test]
+ fn github_web_sibling_checksum_and_notes_links() {
+ let html = r#"<a href="/artempyanykh/marksman/releases/download/2024-12-18/marksman-macos" rel="nofollow">
+ <a href="/artempyanykh/marksman/releases/download/2024-12-18/marksman-macos.sha256" rel="nofollow">"#;
+ let d = github_pick_web(
+ "artempyanykh/marksman",
+ "2024-12-18",
+ html,
+ None,
+ find("marksman").unwrap().asset_rules(),
+ MAC_ARM,
+ )
+ .unwrap();
+ assert_eq!(
+ d.checksum_url.as_deref(),
+ Some("https://github.com/artempyanykh/marksman/releases/download/2024-12-18/marksman-macos.sha256")
+ );
+
+ let notes = r#"<li><a href="https://download-cdn.jetbrains.com/kotlin-lsp/0.253.10629/kotlin-lsp-0.253.10629-mac-aarch64.zip?a=1&amp;b=2" rel="nofollow">macOS arm</a></li>"#;
+ let rules = find("kotlin-lsp").unwrap().asset_rules();
+ let d =
+ github_pick_web("Kotlin/kotlin-lsp", "kotlin-lsp/v0.253.10629", "", Some(notes), rules, MAC_ARM).unwrap();
+ assert_eq!(d.name, "kotlin-lsp-0.253.10629-mac-aarch64.zip");
+ assert_eq!(
+ d.url,
+ "https://download-cdn.jetbrains.com/kotlin-lsp/0.253.10629/kotlin-lsp-0.253.10629-mac-aarch64.zip?a=1&b=2"
+ );
+ }
+
#[test]
fn hashicorp() {
let json = r#"{"version":"0.36.4","url_shasums":"https://r/terraform-ls_0.36.4_SHA256SUMS","builds":[