The IDE you know, rebuilt in Rust. IntelliJ IDEA–style keymap, tool windows and search-everywhere in a fast native app: editor, code insight, debugger, Git, terminal, database and HTTP clients for 60+ languages.
TypeScript code-editorcross-platformdapdatabase-clientdebuggerdesktop-appdeveloper-toolsgithttp-clientideintellijlsprusttauriterminal
fix(tools): install GitHub releases when the API is rate-limited (#1715)
l0ng-aic1aa092d7505
diff --git a/crates/ide-tools/src/lib.rs b/crates/ide-tools/src/lib.rsindex d0a8b65d5..dbdddfa5b 100644--- a/crates/ide-tools/src/lib.rs+++ b/crates/ide-tools/src/lib.rs@@ -1221,10 +1221,26 @@ impl Tools {cancel: &CancellationToken,) -> IdeResult<sources::Download> {match source {- ReleaseSource::GitHub { repo } => {- let json = http.get_text(&sources::github_latest_url(repo), cancel)?;- sources::github_pick(&json, assets, self.platform)- }+ ReleaseSource::GitHub { repo } => match http.get_text(&sources::github_latest_url(repo), cancel) {+ Ok(json) => sources::github_pick(&json, assets, self.platform),+ Err(e) if e.kind == ErrorKind::Cancelled => Err(e),+ // Without a token the API allows 60 requests an hour per IP address, often used+ // up behind a shared one (403/429); the release's web pages have no such limit.+ Err(api) => self.resolve_github_web(http, repo, assets, cancel).map_err(|web| {+ if web.kind == ErrorKind::Cancelled {+ return web;+ }+ tracing::warn!("GitHub release lookup for {repo} failed: {api}; web pages: {web}");+ IdeError::new(+ api.kind,+ format!(+ "{}\nDownload it from {} and set its Path in Settings › Languages & Frameworks › Language Servers.",+ api.message,+ sources::github_latest_page(repo)+ ),+ )+ }),+ },ReleaseSource::HashiCorp { product } => {let json = http.get_text(&sources::hashicorp_latest_url(product), cancel)?;sources::hashicorp_pick(&json, assets, self.platform)@@ -1251,6 +1267,27 @@ impl Tools {}}+ /// [`resolve_release`](Self::resolve_release) for a GitHub release, from its web pages.+ fn resolve_github_web(+ &self,+ http: &Http,+ repo: &str,+ assets: &[platform::AssetRule],+ cancel: &CancellationToken,+ ) -> IdeResult<sources::Download> {+ let page = http.effective_url(&sources::github_latest_page(repo), cancel)?;+ let tag = sources::github_tag_from_url(&page)+ .ok_or_else(|| IdeError::unavailable(format!("{repo} has no published release")))?;+ let list = http.get_text(&sources::github_assets_page(repo, &tag), cancel)?;+ match sources::github_pick_web(repo, &tag, &list, None, assets, self.platform) {+ Err(e) if e.kind == ErrorKind::Unavailable => {+ let notes = http.get_text(&sources::github_tag_page(repo, &tag), cancel)?;+ sources::github_pick_web(repo, &tag, &list, Some(¬es), assets, self.platform)+ }+ result => result,+ }+ }+fn install_release(&self,spec: &ToolSpec,diff --git a/crates/ide-tools/src/process.rs b/crates/ide-tools/src/process.rsindex ac8a48b26..7d34423e4 100644--- a/crates/ide-tools/src/process.rs+++ b/crates/ide-tools/src/process.rs@@ -263,6 +263,15 @@ fn curl_takes_revoke_best_effort(version: &str) -> bool {(major, minor) >= (7, 70) && first.split_whitespace().any(|w| w.eq_ignore_ascii_case("schannel"))}+#[derive(Clone, Copy, PartialEq, Eq)]+enum Request {+ Get,+ /// Adoptium uses 404 as well as an empty array for absent platform builds.+ GetOrEmptyOn404,+ /// Headers, then the URL redirects ended at.+ Head,+}+/// HTTP via the system `curl` (shipped with macOS, Windows 10+ and every/// mainstream Linux), so the IDE carries no TLS stack of its own.pub struct Http {@@ -336,26 +345,39 @@ impl Http {self.get_text_limited(url, None, cancel)}+ /// Where `url` ends up after redirects (a HEAD request).+ pub fn effective_url(&self, url: &str, cancel: &CancellationToken) -> IdeResult<String> {+ let out = self.get_text_response(url, None, Request::Head, cancel)?;+ Ok(out.trim_end().rsplit('\n').next().unwrap_or_default().trim().to_owned())+ }+/// GETs `url` as text, failing when the response is larger than `max_bytes`.pub fn get_text_limited(&self, url: &str, max_bytes: Option<u64>, cancel: &CancellationToken) -> IdeResult<String> {- self.get_text_response(url, max_bytes, false, cancel)+ self.get_text_response(url, max_bytes, Request::Get, cancel)}/// Adoptium uses 404 as well as an empty array for absent platform builds.pub fn get_text_or_empty_on_404(&self, url: &str, cancel: &CancellationToken) -> IdeResult<String> {- self.get_text_response(url, None, true, cancel)+ self.get_text_response(url, None, Request::GetOrEmptyOn404, cancel)}fn get_text_response(&self,url: &str,max_bytes: Option<u64>,- empty_on_404: bool,+ request: Request,cancel: &CancellationToken,) -> IdeResult<String> {let (mut cmd, token) = self.base(url, cancel)?;- if empty_on_404 {- cmd.args(["--write-out", "\n%{http_code}"]);+ let empty_on_404 = request == Request::GetOrEmptyOn404;+ match request {+ Request::Get => {}+ Request::GetOrEmptyOn404 => {+ cmd.args(["--write-out", "\n%{http_code}"]);+ }+ Request::Head => {+ cmd.args(["--head", "--write-out", "\n%{url_effective}"]);+ }}if let Some(max) = max_bytes {cmd.arg("--max-filesize").arg(max.to_string());diff --git a/crates/ide-tools/src/sources.rs b/crates/ide-tools/src/sources.rsindex 2337bc75c..0e5da7481 100644--- a/crates/ide-tools/src/sources.rs+++ b/crates/ide-tools/src/sources.rs@@ -47,38 +47,146 @@ pub fn github_latest_url(repo: &str) -> String {format!("https://api.github.com/repos/{repo}/releases/latest")}+/// The latest release's web page; it redirects to `…/releases/tag/<tag>`. Web pages don't+/// count against the API's rate limit for unauthenticated clients (60 requests an hour).+pub fn github_latest_page(repo: &str) -> String {+ format!("https://github.com/{repo}/releases/latest")+}++/// The tag `github_latest_page` redirected to.+pub fn github_tag_from_url(url: &str) -> Option<String> {+ let (_, tag) = url.split_once("/releases/tag/")?;+ let tag = tag.split(['?', '#']).next().unwrap_or(tag).trim_end_matches('/');+ (!tag.is_empty()).then(|| percent_decode(tag))+}++pub fn github_tag_page(repo: &str, tag: &str) -> String {+ format!("https://github.com/{repo}/releases/tag/{tag}")+}++/// The asset list the release page loads (HTML).+pub fn github_assets_page(repo: &str, tag: &str) -> String {+ format!("https://github.com/{repo}/releases/expanded_assets/{tag}")+}++/// One file of a release, from the API or its web page.+struct ReleaseAsset {+ name: String,+ url: String,+ size: Option<u64>,+ sha256: Option<String>,+}+/// Picks the platform asset from a GitHub `releases/latest` response. Links/// in the release notes count as assets too (some projects host binaries on/// a CDN and only link them).pub fn github_pick(json: &str, rules: &[AssetRule], platform: Platform) -> IdeResult<Download> {let rel: GhRelease = serde_json::from_str(json).map_err(|e| proto(format!("unexpected GitHub response: {e}")))?;- let version = rel.tag_name.trim_start_matches('v').to_string();- let names: Vec<&str> = rel.assets.iter().map(|a| a.name.as_str()).collect();+ let assets: Vec<ReleaseAsset> = rel+ .assets+ .into_iter()+ .map(|a| ReleaseAsset {+ sha256: a.digest.as_deref().and_then(|d| d.strip_prefix("sha256:")).map(str::to_ascii_lowercase),+ name: a.name,+ url: a.browser_download_url,+ size: a.size,+ })+ .collect();+ pick_release(&rel.tag_name, &assets, rel.body.as_deref().unwrap_or(""), rules, platform)+}++/// [`github_pick`] from the release's web pages: `assets_html` from [`github_assets_page`] and,+/// for projects that only link their builds, `notes_html` from [`github_tag_page`].+pub fn github_pick_web(+ repo: &str,+ tag: &str,+ assets_html: &str,+ notes_html: Option<&str>,+ rules: &[AssetRule],+ platform: Platform,+) -> IdeResult<Download> {+ let prefix = format!("href=\"/{repo}/releases/download/");+ let mut assets: Vec<ReleaseAsset> = Vec::new();+ let mut rest = assets_html;+ while let Some(i) = rest.find(&prefix) {+ let tail = &rest[i + "href=\"".len()..];+ let Some(end) = tail.find('"') else { break };+ let path = html_unescape(&tail[..end]);+ rest = &tail[end..];+ let name = percent_decode(file_name(&path));+ if name.is_empty() || assets.iter().any(|a| a.name == name) {+ continue;+ }+ // `<clipboard-copy aria-label="Copy to clipboard digest for <name>" … value="sha256:<hex>">`+ let sha256 = assets_html.find(&format!("digest for {}\"", html_escape(&name))).and_then(|at| {+ let tag_end = assets_html[at..].find('>').map_or(assets_html.len(), |e| at + e);+ let value = assets_html[at..tag_end].split_once("value=\"sha256:")?.1;+ let hex: String = value.chars().take_while(char::is_ascii_hexdigit).collect();+ (hex.len() == 64).then(|| hex.to_ascii_lowercase())+ });+ assets.push(ReleaseAsset { url: format!("https://github.com{path}"), name, size: None, sha256 });+ }+ pick_release(tag, &assets, &html_unescape(notes_html.unwrap_or("")), rules, platform)+}++fn pick_release(+ tag: &str,+ assets: &[ReleaseAsset],+ notes: &str,+ rules: &[AssetRule],+ platform: Platform,+) -> IdeResult<Download> {+ let version = tag.trim_start_matches('v').to_string();+ let names: Vec<&str> = assets.iter().map(|a| a.name.as_str()).collect();if let Some(name) = select_asset(&names, rules, platform) {- let a = rel.assets.iter().find(|a| a.name == name).expect("selected from list");- let sha256 = a.digest.as_deref().and_then(|d| d.strip_prefix("sha256:")).map(str::to_ascii_lowercase);- let checksum_url = rel- .assets+ let a = assets.iter().find(|a| a.name == name).expect("selected from list");+ let checksum_url = assets.iter().find(|c| c.name == format!("{name}.sha256") || c.name == format!("{name}.sha256sum"))- .map(|c| c.browser_download_url.clone());+ .map(|c| c.url.clone());return Ok(Download {version,name: a.name.clone(),- url: a.browser_download_url.clone(),+ url: a.url.clone(),size: a.size,- sha256,+ sha256: a.sha256.clone(),checksum_url,});}- let links = extract_links(rel.body.as_deref().unwrap_or(""));+ let links = extract_links(notes);let link_names: Vec<&str> = links.iter().map(|u| file_name(u)).collect();if let Some(name) = select_asset(&link_names, rules, platform) {let url = links.iter().find(|u| file_name(u) == name).expect("selected from list").clone();let checksum_url = links.iter().find(|u| *u == &format!("{url}.sha256")).cloned();return Ok(Download { version, name: name.to_string(), url, size: None, sha256: None, checksum_url });}- Err(IdeError::unavailable(format!("release {} has no build for this platform", rel.tag_name)))+ Err(IdeError::unavailable(format!("release {tag} has no build for this platform")))+}++fn html_unescape(s: &str) -> String {+ s.replace(""", "\"").replace("'", "'").replace("<", "<").replace(">", ">").replace("&", "&")+}++fn html_escape(s: &str) -> String {+ s.replace('&', "&").replace('"', """).replace('<', "<").replace('>', ">")+}++fn percent_decode(s: &str) -> String {+ let bytes = s.as_bytes();+ let mut out = Vec::with_capacity(bytes.len());+ let mut i = 0;+ while i < bytes.len() {+ if bytes[i] == b'%'+ && let Some(b) = s.get(i + 1..i + 3).and_then(|h| u8::from_str_radix(h, 16).ok())+ {+ out.push(b);+ i += 3;+ } else {+ out.push(bytes[i]);+ i += 1;+ }+ }+ String::from_utf8(out).unwrap_or_else(|_| s.to_owned())}fn file_name(url: &str) -> &str {@@ -336,6 +444,71 @@ mod tests {assert_eq!(d.name, "kotlin-lsp-0.253.10629-linux-x64.zip");}+ #[test]+ fn github_tag_from_redirect() {+ let url = "https://github.com/Kotlin/kotlin-lsp/releases/tag/kotlin-lsp/v263.7084.0";+ assert_eq!(github_tag_from_url(url).as_deref(), Some("kotlin-lsp/v263.7084.0"));+ let url = "https://github.com/o/r/releases/tag/v1.0%2Bbuild";+ assert_eq!(github_tag_from_url(url).as_deref(), Some("v1.0+build"));+ // A repository without releases stays on the list.+ assert_eq!(github_tag_from_url("https://github.com/o/r/releases"), None);+ }++ /// Trimmed from `github.com/<repo>/releases/expanded_assets/<tag>`.+ const ASSETS_HTML: &str = r#"<li><div><a href="/LuaLS/lua-language-server/releases/download/3.13.5/lua-language-server-3.13.5-darwin-arm64.tar.gz" rel="nofollow" data-turbo="false" class="wb-break-all">+ <span class="text-bold">lua-language-server-3.13.5-darwin-arm64.tar.gz</span></a></div>+ <span class="Truncate-text">sha256:ABCDEF0123456789abcdef0123456789abcdef0123456789abcdef0123456789</span>+ <clipboard-copy id="clipboard-button-sha256:abc" aria-label="Copy to clipboard digest for lua-language-server-3.13.5-darwin-arm64.tar.gz" type="button" value="sha256:ABCDEF0123456789abcdef0123456789abcdef0123456789abcdef0123456789" class="Button"></clipboard-copy></li>+ <li><a href="/LuaLS/lua-language-server/releases/download/3.13.5/lua-language-server-3.13.5-linux-x64.tar.gz" rel="nofollow">lua-language-server-3.13.5-linux-x64.tar.gz</a></li>+ <li><a href="/LuaLS/lua-language-server/archive/refs/tags/3.13.5.zip" rel="nofollow">Source code (zip)</a></li>"#;++ #[test]+ fn github_web_assets_with_digest() {+ let rules = find("lua-language-server").unwrap().asset_rules();+ let d = github_pick_web("LuaLS/lua-language-server", "3.13.5", ASSETS_HTML, None, rules, MAC_ARM).unwrap();+ assert_eq!(d.version, "3.13.5");+ assert_eq!(d.name, "lua-language-server-3.13.5-darwin-arm64.tar.gz");+ assert_eq!(+ d.url,+ "https://github.com/LuaLS/lua-language-server/releases/download/3.13.5/lua-language-server-3.13.5-darwin-arm64.tar.gz"+ );+ assert_eq!(d.sha256.as_deref(), Some("abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789"));+ let d = github_pick_web("LuaLS/lua-language-server", "3.13.5", ASSETS_HTML, None, rules, LINUX).unwrap();+ assert_eq!(d.sha256, None);+ let win = Platform { os: Os::Windows, arch: Arch::X64 };+ let err = github_pick_web("LuaLS/lua-language-server", "3.13.5", ASSETS_HTML, None, rules, win).unwrap_err();+ assert_eq!(err.kind, ErrorKind::Unavailable);+ }++ #[test]+ fn github_web_sibling_checksum_and_notes_links() {+ let html = r#"<a href="/artempyanykh/marksman/releases/download/2024-12-18/marksman-macos" rel="nofollow">+ <a href="/artempyanykh/marksman/releases/download/2024-12-18/marksman-macos.sha256" rel="nofollow">"#;+ let d = github_pick_web(+ "artempyanykh/marksman",+ "2024-12-18",+ html,+ None,+ find("marksman").unwrap().asset_rules(),+ MAC_ARM,+ )+ .unwrap();+ assert_eq!(+ d.checksum_url.as_deref(),+ Some("https://github.com/artempyanykh/marksman/releases/download/2024-12-18/marksman-macos.sha256")+ );++ let notes = r#"<li><a href="https://download-cdn.jetbrains.com/kotlin-lsp/0.253.10629/kotlin-lsp-0.253.10629-mac-aarch64.zip?a=1&b=2" rel="nofollow">macOS arm</a></li>"#;+ let rules = find("kotlin-lsp").unwrap().asset_rules();+ let d =+ github_pick_web("Kotlin/kotlin-lsp", "kotlin-lsp/v0.253.10629", "", Some(notes), rules, MAC_ARM).unwrap();+ assert_eq!(d.name, "kotlin-lsp-0.253.10629-mac-aarch64.zip");+ assert_eq!(+ d.url,+ "https://download-cdn.jetbrains.com/kotlin-lsp/0.253.10629/kotlin-lsp-0.253.10629-mac-aarch64.zip?a=1&b=2"+ );+ }+#[test]fn hashicorp() {let json = r#"{"version":"0.36.4","url_shasums":"https://r/terraform-ls_0.36.4_SHA256SUMS","builds":[