From 0b4d53f2d87de452ece089b291a4cae842706c10 Mon Sep 17 00:00:00 2001 From: l0ng-ai <24760907+l0ng-ai@users.noreply.github.com> Date: Fri, 6 Nov 2026 13:23:13 +0800 Subject: [PATCH] fix(tools): install GitHub releases when the API is rate-limited Language servers and debuggers published as GitHub releases were looked up through api.github.com, which allows 60 unauthenticated requests an hour per IP address. Behind a shared address the limit is often used up and installs failed with 403. When the API request fails, the release is now resolved from its web pages instead: the latest-release redirect gives the tag, the asset list gives the files and their SHA-256 digests, and the release notes give linked builds. If that fails too, the error names the release page to download from and the setting to point at the file. Fixes #1651 --- crates/ide-tools/src/lib.rs | 45 +++++++- crates/ide-tools/src/process.rs | 32 +++++- crates/ide-tools/src/sources.rs | 195 ++++++++++++++++++++++++++++++-- 3 files changed, 252 insertions(+), 20 deletions(-) diff --git a/crates/ide-tools/src/lib.rs b/crates/ide-tools/src/lib.rs index d0a8b65d5..dbdddfa5b 100644 --- a/crates/ide-tools/src/lib.rs +++ b/crates/ide-tools/src/lib.rs @@ -1221,10 +1221,26 @@ impl Tools { cancel: &CancellationToken, ) -> IdeResult { match source { - ReleaseSource::GitHub { repo } => { - let json = http.get_text(&sources::github_latest_url(repo), cancel)?; - sources::github_pick(&json, assets, self.platform) - } + ReleaseSource::GitHub { repo } => match http.get_text(&sources::github_latest_url(repo), cancel) { + Ok(json) => sources::github_pick(&json, assets, self.platform), + Err(e) if e.kind == ErrorKind::Cancelled => Err(e), + // Without a token the API allows 60 requests an hour per IP address, often used + // up behind a shared one (403/429); the release's web pages have no such limit. + Err(api) => self.resolve_github_web(http, repo, assets, cancel).map_err(|web| { + if web.kind == ErrorKind::Cancelled { + return web; + } + tracing::warn!("GitHub release lookup for {repo} failed: {api}; web pages: {web}"); + IdeError::new( + api.kind, + format!( + "{}\nDownload it from {} and set its Path in Settings › Languages & Frameworks › Language Servers.", + api.message, + sources::github_latest_page(repo) + ), + ) + }), + }, ReleaseSource::HashiCorp { product } => { let json = http.get_text(&sources::hashicorp_latest_url(product), cancel)?; sources::hashicorp_pick(&json, assets, self.platform) @@ -1251,6 +1267,27 @@ impl Tools { } } + /// [`resolve_release`](Self::resolve_release) for a GitHub release, from its web pages. + fn resolve_github_web( + &self, + http: &Http, + repo: &str, + assets: &[platform::AssetRule], + cancel: &CancellationToken, + ) -> IdeResult { + let page = http.effective_url(&sources::github_latest_page(repo), cancel)?; + let tag = sources::github_tag_from_url(&page) + .ok_or_else(|| IdeError::unavailable(format!("{repo} has no published release")))?; + let list = http.get_text(&sources::github_assets_page(repo, &tag), cancel)?; + match sources::github_pick_web(repo, &tag, &list, None, assets, self.platform) { + Err(e) if e.kind == ErrorKind::Unavailable => { + let notes = http.get_text(&sources::github_tag_page(repo, &tag), cancel)?; + sources::github_pick_web(repo, &tag, &list, Some(¬es), assets, self.platform) + } + result => result, + } + } + fn install_release( &self, spec: &ToolSpec, diff --git a/crates/ide-tools/src/process.rs b/crates/ide-tools/src/process.rs index ac8a48b26..7d34423e4 100644 --- a/crates/ide-tools/src/process.rs +++ b/crates/ide-tools/src/process.rs @@ -263,6 +263,15 @@ fn curl_takes_revoke_best_effort(version: &str) -> bool { (major, minor) >= (7, 70) && first.split_whitespace().any(|w| w.eq_ignore_ascii_case("schannel")) } +#[derive(Clone, Copy, PartialEq, Eq)] +enum Request { + Get, + /// Adoptium uses 404 as well as an empty array for absent platform builds. + GetOrEmptyOn404, + /// Headers, then the URL redirects ended at. + Head, +} + /// HTTP via the system `curl` (shipped with macOS, Windows 10+ and every /// mainstream Linux), so the IDE carries no TLS stack of its own. pub struct Http { @@ -336,26 +345,39 @@ impl Http { self.get_text_limited(url, None, cancel) } + /// Where `url` ends up after redirects (a HEAD request). + pub fn effective_url(&self, url: &str, cancel: &CancellationToken) -> IdeResult { + let out = self.get_text_response(url, None, Request::Head, cancel)?; + Ok(out.trim_end().rsplit('\n').next().unwrap_or_default().trim().to_owned()) + } + /// GETs `url` as text, failing when the response is larger than `max_bytes`. pub fn get_text_limited(&self, url: &str, max_bytes: Option, cancel: &CancellationToken) -> IdeResult { - self.get_text_response(url, max_bytes, false, cancel) + self.get_text_response(url, max_bytes, Request::Get, cancel) } /// Adoptium uses 404 as well as an empty array for absent platform builds. pub fn get_text_or_empty_on_404(&self, url: &str, cancel: &CancellationToken) -> IdeResult { - self.get_text_response(url, None, true, cancel) + self.get_text_response(url, None, Request::GetOrEmptyOn404, cancel) } fn get_text_response( &self, url: &str, max_bytes: Option, - empty_on_404: bool, + request: Request, cancel: &CancellationToken, ) -> IdeResult { let (mut cmd, token) = self.base(url, cancel)?; - if empty_on_404 { - cmd.args(["--write-out", "\n%{http_code}"]); + let empty_on_404 = request == Request::GetOrEmptyOn404; + match request { + Request::Get => {} + Request::GetOrEmptyOn404 => { + cmd.args(["--write-out", "\n%{http_code}"]); + } + Request::Head => { + cmd.args(["--head", "--write-out", "\n%{url_effective}"]); + } } if let Some(max) = max_bytes { cmd.arg("--max-filesize").arg(max.to_string()); diff --git a/crates/ide-tools/src/sources.rs b/crates/ide-tools/src/sources.rs index 2337bc75c..0e5da7481 100644 --- a/crates/ide-tools/src/sources.rs +++ b/crates/ide-tools/src/sources.rs @@ -47,38 +47,146 @@ pub fn github_latest_url(repo: &str) -> String { format!("https://api.github.com/repos/{repo}/releases/latest") } +/// The latest release's web page; it redirects to `…/releases/tag/`. Web pages don't +/// count against the API's rate limit for unauthenticated clients (60 requests an hour). +pub fn github_latest_page(repo: &str) -> String { + format!("https://github.com/{repo}/releases/latest") +} + +/// The tag `github_latest_page` redirected to. +pub fn github_tag_from_url(url: &str) -> Option { + let (_, tag) = url.split_once("/releases/tag/")?; + let tag = tag.split(['?', '#']).next().unwrap_or(tag).trim_end_matches('/'); + (!tag.is_empty()).then(|| percent_decode(tag)) +} + +pub fn github_tag_page(repo: &str, tag: &str) -> String { + format!("https://github.com/{repo}/releases/tag/{tag}") +} + +/// The asset list the release page loads (HTML). +pub fn github_assets_page(repo: &str, tag: &str) -> String { + format!("https://github.com/{repo}/releases/expanded_assets/{tag}") +} + +/// One file of a release, from the API or its web page. +struct ReleaseAsset { + name: String, + url: String, + size: Option, + sha256: Option, +} + /// Picks the platform asset from a GitHub `releases/latest` response. Links /// in the release notes count as assets too (some projects host binaries on /// a CDN and only link them). pub fn github_pick(json: &str, rules: &[AssetRule], platform: Platform) -> IdeResult { let rel: GhRelease = serde_json::from_str(json).map_err(|e| proto(format!("unexpected GitHub response: {e}")))?; - let version = rel.tag_name.trim_start_matches('v').to_string(); - let names: Vec<&str> = rel.assets.iter().map(|a| a.name.as_str()).collect(); + let assets: Vec = rel + .assets + .into_iter() + .map(|a| ReleaseAsset { + sha256: a.digest.as_deref().and_then(|d| d.strip_prefix("sha256:")).map(str::to_ascii_lowercase), + name: a.name, + url: a.browser_download_url, + size: a.size, + }) + .collect(); + pick_release(&rel.tag_name, &assets, rel.body.as_deref().unwrap_or(""), rules, platform) +} + +/// [`github_pick`] from the release's web pages: `assets_html` from [`github_assets_page`] and, +/// for projects that only link their builds, `notes_html` from [`github_tag_page`]. +pub fn github_pick_web( + repo: &str, + tag: &str, + assets_html: &str, + notes_html: Option<&str>, + rules: &[AssetRule], + platform: Platform, +) -> IdeResult { + let prefix = format!("href=\"/{repo}/releases/download/"); + let mut assets: Vec = Vec::new(); + let mut rest = assets_html; + while let Some(i) = rest.find(&prefix) { + let tail = &rest[i + "href=\"".len()..]; + let Some(end) = tail.find('"') else { break }; + let path = html_unescape(&tail[..end]); + rest = &tail[end..]; + let name = percent_decode(file_name(&path)); + if name.is_empty() || assets.iter().any(|a| a.name == name) { + continue; + } + // `` + let sha256 = assets_html.find(&format!("digest for {}\"", html_escape(&name))).and_then(|at| { + let tag_end = assets_html[at..].find('>').map_or(assets_html.len(), |e| at + e); + let value = assets_html[at..tag_end].split_once("value=\"sha256:")?.1; + let hex: String = value.chars().take_while(char::is_ascii_hexdigit).collect(); + (hex.len() == 64).then(|| hex.to_ascii_lowercase()) + }); + assets.push(ReleaseAsset { url: format!("https://github.com{path}"), name, size: None, sha256 }); + } + pick_release(tag, &assets, &html_unescape(notes_html.unwrap_or("")), rules, platform) +} + +fn pick_release( + tag: &str, + assets: &[ReleaseAsset], + notes: &str, + rules: &[AssetRule], + platform: Platform, +) -> IdeResult { + let version = tag.trim_start_matches('v').to_string(); + let names: Vec<&str> = assets.iter().map(|a| a.name.as_str()).collect(); if let Some(name) = select_asset(&names, rules, platform) { - let a = rel.assets.iter().find(|a| a.name == name).expect("selected from list"); - let sha256 = a.digest.as_deref().and_then(|d| d.strip_prefix("sha256:")).map(str::to_ascii_lowercase); - let checksum_url = rel - .assets + let a = assets.iter().find(|a| a.name == name).expect("selected from list"); + let checksum_url = assets .iter() .find(|c| c.name == format!("{name}.sha256") || c.name == format!("{name}.sha256sum")) - .map(|c| c.browser_download_url.clone()); + .map(|c| c.url.clone()); return Ok(Download { version, name: a.name.clone(), - url: a.browser_download_url.clone(), + url: a.url.clone(), size: a.size, - sha256, + sha256: a.sha256.clone(), checksum_url, }); } - let links = extract_links(rel.body.as_deref().unwrap_or("")); + let links = extract_links(notes); let link_names: Vec<&str> = links.iter().map(|u| file_name(u)).collect(); if let Some(name) = select_asset(&link_names, rules, platform) { let url = links.iter().find(|u| file_name(u) == name).expect("selected from list").clone(); let checksum_url = links.iter().find(|u| *u == &format!("{url}.sha256")).cloned(); return Ok(Download { version, name: name.to_string(), url, size: None, sha256: None, checksum_url }); } - Err(IdeError::unavailable(format!("release {} has no build for this platform", rel.tag_name))) + Err(IdeError::unavailable(format!("release {tag} has no build for this platform"))) +} + +fn html_unescape(s: &str) -> String { + s.replace(""", "\"").replace("'", "'").replace("<", "<").replace(">", ">").replace("&", "&") +} + +fn html_escape(s: &str) -> String { + s.replace('&', "&").replace('"', """).replace('<', "<").replace('>', ">") +} + +fn percent_decode(s: &str) -> String { + let bytes = s.as_bytes(); + let mut out = Vec::with_capacity(bytes.len()); + let mut i = 0; + while i < bytes.len() { + if bytes[i] == b'%' + && let Some(b) = s.get(i + 1..i + 3).and_then(|h| u8::from_str_radix(h, 16).ok()) + { + out.push(b); + i += 3; + } else { + out.push(bytes[i]); + i += 1; + } + } + String::from_utf8(out).unwrap_or_else(|_| s.to_owned()) } fn file_name(url: &str) -> &str { @@ -336,6 +444,71 @@ mod tests { assert_eq!(d.name, "kotlin-lsp-0.253.10629-linux-x64.zip"); } + #[test] + fn github_tag_from_redirect() { + let url = "https://github.com/Kotlin/kotlin-lsp/releases/tag/kotlin-lsp/v263.7084.0"; + assert_eq!(github_tag_from_url(url).as_deref(), Some("kotlin-lsp/v263.7084.0")); + let url = "https://github.com/o/r/releases/tag/v1.0%2Bbuild"; + assert_eq!(github_tag_from_url(url).as_deref(), Some("v1.0+build")); + // A repository without releases stays on the list. + assert_eq!(github_tag_from_url("https://github.com/o/r/releases"), None); + } + + /// Trimmed from `github.com//releases/expanded_assets/`. + const ASSETS_HTML: &str = r#"
  • + sha256:ABCDEF0123456789abcdef0123456789abcdef0123456789abcdef0123456789 +
    • +
  • lua-language-server-3.13.5-linux-x64.tar.gz
    • +
  • Source code (zip)
    • "#; + + #[test] + fn github_web_assets_with_digest() { + let rules = find("lua-language-server").unwrap().asset_rules(); + let d = github_pick_web("LuaLS/lua-language-server", "3.13.5", ASSETS_HTML, None, rules, MAC_ARM).unwrap(); + assert_eq!(d.version, "3.13.5"); + assert_eq!(d.name, "lua-language-server-3.13.5-darwin-arm64.tar.gz"); + assert_eq!( + d.url, + "https://github.com/LuaLS/lua-language-server/releases/download/3.13.5/lua-language-server-3.13.5-darwin-arm64.tar.gz" + ); + assert_eq!(d.sha256.as_deref(), Some("abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789")); + let d = github_pick_web("LuaLS/lua-language-server", "3.13.5", ASSETS_HTML, None, rules, LINUX).unwrap(); + assert_eq!(d.sha256, None); + let win = Platform { os: Os::Windows, arch: Arch::X64 }; + let err = github_pick_web("LuaLS/lua-language-server", "3.13.5", ASSETS_HTML, None, rules, win).unwrap_err(); + assert_eq!(err.kind, ErrorKind::Unavailable); + } + + #[test] + fn github_web_sibling_checksum_and_notes_links() { + let html = r#" + "#; + let d = github_pick_web( + "artempyanykh/marksman", + "2024-12-18", + html, + None, + find("marksman").unwrap().asset_rules(), + MAC_ARM, + ) + .unwrap(); + assert_eq!( + d.checksum_url.as_deref(), + Some("https://github.com/artempyanykh/marksman/releases/download/2024-12-18/marksman-macos.sha256") + ); + + let notes = r#"
  • macOS arm
    • "#; + let rules = find("kotlin-lsp").unwrap().asset_rules(); + let d = + github_pick_web("Kotlin/kotlin-lsp", "kotlin-lsp/v0.253.10629", "", Some(notes), rules, MAC_ARM).unwrap(); + assert_eq!(d.name, "kotlin-lsp-0.253.10629-mac-aarch64.zip"); + assert_eq!( + d.url, + "https://download-cdn.jetbrains.com/kotlin-lsp/0.253.10629/kotlin-lsp-0.253.10629-mac-aarch64.zip?a=1&b=2" + ); + } + #[test] fn hashicorp() { let json = r#"{"version":"0.36.4","url_shasums":"https://r/terraform-ls_0.36.4_SHA256SUMS","builds":[