From 3407195dc06c57d40b644e70b7cbcb959cacfed0 Mon Sep 17 00:00:00 2001 From: l0ng-ai <24760907+l0ng-ai@users.noreply.github.com> Date: Sun, 8 Nov 2026 00:05:41 +0800 Subject: [PATCH] fix(libraries): discover dependencies across project modules (#2022) Fixes #1977. External Libraries only inspected the project root and cached incomplete results, so ecosystems in nested modules/workspace members (e.g. a pnpm frontend under a Cargo + Maven repo) and dependencies installed after the first listing never appeared. Providers now discover module/workspace roots (package.json workspaces, pnpm-workspace.yaml, Cargo members, nested manifests), follow pnpm symlinks, include local Cargo path dependencies, stream groups to the Project view as each provider finishes so a slow cargo metadata doesn't hide the others, log failures instead of dropping them, and don't cache cancelled or failed results. --- crates/ide-core/src/project_roots.rs | 78 +++ crates/ide-host/src/ops/sdk.rs | 65 +- crates/ide-run/src/detect.rs | 57 +- crates/ide-sdk/src/exec.rs | 58 +- crates/ide-sdk/src/libraries.rs | 577 ++++++++++++++++-- crates/ide-sdk/src/tests.rs | 47 +- src-tauri/src/commands/sdk.rs | 5 +- src/bindings.ts | 2 +- src/features/libraries/index.ts | 67 +- src/features/libraries/libraries.test.ts | 26 + .../languages/ProjectStructurePage.tsx | 19 +- src/platform/libraries.ts | 17 + 12 files changed, 864 insertions(+), 154 deletions(-) create mode 100644 src/platform/libraries.ts diff --git a/crates/ide-core/src/project_roots.rs b/crates/ide-core/src/project_roots.rs index 458c9e96d..58ccaa822 100644 --- a/crates/ide-core/src/project_roots.rs +++ b/crates/ide-core/src/project_roots.rs @@ -1,5 +1,6 @@ //! Shared filesystem rules for discovering builds and resolving their owners. +use std::collections::HashSet; use std::path::{Component, Path, PathBuf}; pub const SEARCH_DEPTH: usize = 3; @@ -35,10 +36,23 @@ pub fn discover_matching( max_depth: usize, stop_at_match: bool, marked: impl Fn(&Path) -> bool, +) -> Vec { + discover_matching_cancellable(root, max_depth, stop_at_match, &crate::CancellationToken::new(), marked) +} + +pub fn discover_matching_cancellable( + root: &Path, + max_depth: usize, + stop_at_match: bool, + cancel: &crate::CancellationToken, + marked: impl Fn(&Path) -> bool, ) -> Vec { let mut found = Vec::new(); let mut pending = vec![(root.to_path_buf(), 0)]; while let Some((dir, depth)) = pending.pop() { + if cancel.is_cancelled() { + break; + } if marked(&dir) { found.push(dir.clone()); if stop_at_match { @@ -66,6 +80,70 @@ pub fn discover_matching( found } +/// Expands workspace member patterns one level deep: `dir`, `dir/*`, `dir/**`. +pub fn expand_members(root: &Path, patterns: &[String], marker: &str) -> Vec { + let mut dirs = Vec::new(); + for pattern in patterns { + let pattern = pattern.trim().trim_start_matches("./").trim_end_matches('/'); + if pattern.is_empty() || pattern.starts_with('!') { + continue; + } + let base = pattern.strip_suffix("/**").or_else(|| pattern.strip_suffix("/*")); + match base { + Some(base) if !base.contains('*') => { + dirs.extend(children(&root.join(base)).into_iter().filter(|d| d.join(marker).is_file())) + } + None if !pattern.contains('*') => { + let dir = root.join(pattern); + if dir.join(marker).is_file() { + dirs.push(dir); + } + } + _ => {} + } + } + let mut seen = HashSet::new(); + dirs.retain(|d| seen.insert(d.clone())); + dirs +} + +pub fn node_workspace_members(root: &Path) -> Vec { + let package = std::fs::read(root.join("package.json")) + .ok() + .and_then(|bytes| serde_json::from_slice::(&bytes).ok()) + .unwrap_or_default(); + expand_members(root, &node_workspace_patterns(root, &package), "package.json") +} + +fn node_workspace_patterns(root: &Path, pkg: &serde_json::Value) -> Vec { + let mut patterns: Vec = Vec::new(); + let ws = pkg.get("workspaces"); + let list = ws + .and_then(serde_json::Value::as_array) + .or_else(|| ws.and_then(|w| w.get("packages")).and_then(serde_json::Value::as_array)); + if let Some(list) = list { + patterns.extend(list.iter().filter_map(serde_json::Value::as_str).map(str::to_owned)); + } + if let Ok(yaml) = std::fs::read_to_string(root.join("pnpm-workspace.yaml")) { + let mut in_packages = false; + for line in yaml.lines() { + let trimmed = line.trim(); + if trimmed.is_empty() || trimmed.starts_with('#') { + continue; + } + if !line.starts_with([' ', '\t', '-']) { + in_packages = trimmed.starts_with("packages:"); + continue; + } + if in_packages && let Some(item) = trimmed.strip_prefix('-') { + let item = item.split(" #").next().unwrap_or_default().trim().trim_matches(['"', '\'']); + patterns.push(item.to_owned()); + } + } + } + patterns +} + /// The closest package manager declaration or lockfile, shared by build and run. pub fn package_manager(root: &Path) -> &'static str { for dir in root.ancestors() { diff --git a/crates/ide-host/src/ops/sdk.rs b/crates/ide-host/src/ops/sdk.rs index 46db0eaad..e9d2fb79b 100644 --- a/crates/ide-host/src/ops/sdk.rs +++ b/crates/ide-host/src/ops/sdk.rs @@ -149,25 +149,56 @@ pub async fn sdk_php_composer_install( /// SDK / library source folders shown under "External Libraries" in the Project view, /// with the Maven / Gradle dependency jars of the build model. -pub async fn sdk_project_libraries(host: &Host, window: &str) -> IdeResult> { +pub async fn sdk_project_libraries( + host: &Host, + window: &str, + request: RequestId, + on_groups: Channel>, +) -> IdeResult> { host.require_trust(window)?; let project = host.project(window)?; let (root, cache, settings) = (project.root.clone(), project.build_cache_dir(), project.settings.effective()); - tokio::task::spawn_blocking(move || { - let model = crate::ops::build::load(&root, &cache, &settings); - let deps: Vec = ide_build::dependency_coordinates(&root, &model) - .into_iter() - .map(|c| ide_sdk::libraries::BuildDependency { - gradle: c.gradle, - group: c.group_id, - artifact: c.artifact_id, - version: c.version, + let cancel = host.begin_project_request(window, request); + let libraries = { + let (root, settings, cancel, on_groups) = (root.clone(), settings.clone(), cancel.clone(), on_groups.clone()); + tokio::task::spawn_blocking(move || { + ide_sdk::libraries::project_libraries_stream(&root, &settings, &[], &cancel, &|groups| { + let _ = on_groups.send(groups); }) - .collect(); - ide_sdk::libraries::project_libraries(&root, &settings, &deps) - }) - .await - .map_err(join_err) + }) + }; + let dependencies = { + let cancel = cancel.clone(); + tokio::task::spawn_blocking(move || { + if cancel.is_cancelled() { + return Vec::new(); + } + let model = crate::ops::build::load(&root, &cache, &settings); + let deps: Vec = ide_build::dependency_coordinates(&root, &model) + .into_iter() + .map(|c| ide_sdk::libraries::BuildDependency { + gradle: c.gradle, + group: c.group_id, + artifact: c.artifact_id, + version: c.version, + }) + .collect(); + if cancel.is_cancelled() { + return Vec::new(); + } + let groups = ide_sdk::libraries::dependency_groups(&root, &settings, &deps); + if !cancel.is_cancelled() { + let _ = on_groups.send(groups.clone()); + } + groups + }) + }; + let (libraries, dependencies) = tokio::join!(libraries, dependencies); + host.end_request(request); + if cancel.is_cancelled() { + return Err(IdeError::cancelled()); + } + Ok(ide_sdk::libraries::merge_groups([libraries.map_err(join_err)?, dependencies.map_err(join_err)?].concat())) } /// Looks up `names` (bare command names): `node_modules/.bin` above `near`, then the search path. @@ -268,7 +299,9 @@ pub(crate) fn register(d: &mut Dispatcher) { "sdk_php_composer_install", |c| async move { sdk_php_composer_install(&c.host, &c.window, c.arg("path")?).await }, ); - d.add("sdk_project_libraries", |c| async move { sdk_project_libraries(&c.host, &c.window).await }); + d.add("sdk_project_libraries", |c| async move { + sdk_project_libraries(&c.host, &c.window, c.arg("request")?, c.sender("onGroups")?).await + }); d.add("sdk_locate_tools", |c| async move { sdk_locate_tools(&c.host, &c.window, c.arg("names")?, c.arg("near")?).await }); diff --git a/crates/ide-run/src/detect.rs b/crates/ide-run/src/detect.rs index 24120a724..7f7c714fe 100644 --- a/crates/ide-run/src/detect.rs +++ b/crates/ide-run/src/detect.rs @@ -7,6 +7,7 @@ pub(crate) mod cmake; mod dotnet; use crate::RunConfig; +use ide_core::project_roots::expand_members; use serde_json::Value as Json; use std::collections::{BTreeMap, HashSet}; use std::fs; @@ -213,33 +214,6 @@ fn subdirs(dir: &Path) -> Vec { ide_core::project_roots::children(dir) } -/// Expands workspace member patterns one level deep: `dir`, `dir/*`, `dir/**`. -fn expand_members(root: &Path, patterns: &[String], marker: &str) -> Vec { - let mut dirs = Vec::new(); - for pattern in patterns { - let pattern = pattern.trim().trim_start_matches("./").trim_end_matches('/'); - if pattern.is_empty() || pattern.starts_with('!') { - continue; - } - let base = pattern.strip_suffix("/**").or_else(|| pattern.strip_suffix("/*")); - match base { - Some(base) if !base.contains('*') => { - dirs.extend(subdirs(&root.join(base)).into_iter().filter(|d| d.join(marker).is_file())) - } - None if !pattern.contains('*') => { - let dir = root.join(pattern); - if dir.join(marker).is_file() { - dirs.push(dir); - } - } - _ => {} - } - } - let mut seen = HashSet::new(); - dirs.retain(|d| seen.insert(d.clone())); - dirs -} - fn relative(root: &Path, dir: &Path) -> PathBuf { dir.strip_prefix(root).map(Path::to_path_buf).unwrap_or_else(|_| dir.to_path_buf()) } @@ -272,33 +246,6 @@ fn push_scripts(pkg: &Json, pm: &str, prefix: Option<&str>, cwd: Option } } -fn node_workspace_patterns(root: &Path, pkg: &Json) -> Vec { - let mut patterns: Vec = Vec::new(); - let ws = pkg.get("workspaces"); - let list = ws.and_then(Json::as_array).or_else(|| ws.and_then(|w| w.get("packages")).and_then(Json::as_array)); - if let Some(list) = list { - patterns.extend(list.iter().filter_map(Json::as_str).map(str::to_owned)); - } - if let Some(yaml) = read(&root.join("pnpm-workspace.yaml")) { - let mut in_packages = false; - for line in yaml.lines() { - let trimmed = line.trim(); - if trimmed.is_empty() || trimmed.starts_with('#') { - continue; - } - if !line.starts_with([' ', '\t', '-']) { - in_packages = trimmed.starts_with("packages:"); - continue; - } - if in_packages && let Some(item) = trimmed.strip_prefix('-') { - let item = item.split(" #").next().unwrap_or_default().trim().trim_matches(['"', '\'']); - patterns.push(item.to_owned()); - } - } - } - patterns -} - fn detect_node(root: &Path, out: &mut Vec) { let mut covered = HashSet::new(); let projects = @@ -314,7 +261,7 @@ fn detect_node(root: &Path, out: &mut Vec) { let rel = relative(root, &dir); let label = ide_core::ipc_path::to_ipc(&rel); push_scripts(&pkg, pm, (dir != root).then_some(label.as_str()), (dir != root).then_some(rel), out); - for member_dir in expand_members(&dir, &node_workspace_patterns(&dir, &pkg), "package.json") { + for member_dir in ide_core::project_roots::node_workspace_members(&dir) { if !covered.insert(member_dir.clone()) { continue; } diff --git a/crates/ide-sdk/src/exec.rs b/crates/ide-sdk/src/exec.rs index 26a791f91..b62278a32 100644 --- a/crates/ide-sdk/src/exec.rs +++ b/crates/ide-sdk/src/exec.rs @@ -61,14 +61,43 @@ pub(crate) fn query_in( /// [`query`] of a command built by the caller; `PATH` is the search path unless it sets one. pub(crate) fn query_command(mut cmd: Command, timeout: Duration) -> Option { + query_command_cancellable(&mut cmd, timeout, None) +} + +pub(crate) fn query_in_cancellable( + program: &Path, + args: &[&str], + cwd: &Path, + env: &[(&str, &str)], + timeout: Duration, + cancel: &CancellationToken, +) -> Option { + let mut cmd = Command::new(program); + cmd.args(args).current_dir(cwd).envs(env.iter().copied()); + query_command_cancellable(&mut cmd, timeout, Some(cancel)) +} + +fn query_command_cancellable( + cmd: &mut Command, + timeout: Duration, + cancel: Option<&CancellationToken>, +) -> Option { + if cancel.is_some_and(CancellationToken::is_cancelled) { + return None; + } if !cmd.get_envs().any(|(k, _)| k == "PATH") { cmd.env("PATH", search_path()); } - let mut child = ide_core::process::prepare(&mut cmd) + let mut child = ide_core::process::prepare(cmd) .stdin(Stdio::null()) .stdout(Stdio::piped()) .stderr(Stdio::piped()) .spawn() + .inspect_err(|error| { + if cancel.is_some() { + tracing::warn!(?cmd, %error, "cannot query external libraries"); + } + }) .ok()?; // Chunks as they come, `None` at the end of a pipe. A reader stops once the query // has returned, closing the pipe on whatever still writes to it. @@ -96,35 +125,54 @@ pub(crate) fn query_command(mut cmd: Command, timeout: Duration) -> Option break status, Ok(None) if Instant::now() < deadline => std::thread::sleep(Duration::from_millis(5)), _ => { let _ = child.kill(); let _ = child.wait(); + if cancel.is_some() { + tracing::warn!(?cmd, "external libraries query timed out or could not be polled"); + } return None; } } }; - if !status.success() { + if !status.success() && cancel.is_none() { return None; } // Up to the deadline, or briefly past it for a program that exited just before: // a process it left running (a wrapper's background job) can hold the pipes // open, or keep writing to them, after it exits. - let until = deadline.max(Instant::now() + Duration::from_millis(100)); + let drain_deadline = Instant::now() + Duration::from_millis(100); + let until = if status.success() { deadline.max(drain_deadline) } else { drain_deadline }; let (mut out, mut err, mut open) = (Vec::new(), Vec::new(), 2); while open > 0 { + if cancel.is_some_and(CancellationToken::is_cancelled) { + return None; + } let left = until.saturating_duration_since(Instant::now()); if left.is_zero() { break; } - match rx.recv_timeout(left) { + match rx.recv_timeout(left.min(Duration::from_millis(20))) { Ok((is_out, Some(chunk))) => if is_out { &mut out } else { &mut err }.extend(chunk), Ok((_, None)) => open -= 1, - Err(_) => break, + Err(mpsc::RecvTimeoutError::Timeout) => continue, + Err(mpsc::RecvTimeoutError::Disconnected) => break, } } + if !status.success() { + if cancel.is_some() { + tracing::warn!(?cmd, %status, stderr = %decode_tool_output(&err), "external libraries query failed"); + } + return None; + } let text = if out.trim_ascii().is_empty() { err } else { out }; Some(decode_tool_output(&text).trim().to_owned()) } diff --git a/crates/ide-sdk/src/libraries.rs b/crates/ide-sdk/src/libraries.rs index 58d77c989..0e8c3e403 100644 --- a/crates/ide-sdk/src/libraries.rs +++ b/crates/ide-sdk/src/libraries.rs @@ -4,14 +4,15 @@ //! the JDK, `node_modules` packages, Cargo crates and the Rust standard //! library, GOROOT and the Go module cache. //! -//! Each ecosystem is detected only when the project has its manifest, and its -//! result is cached until the manifests (or the relevant settings) change. +//! Ecosystems are detected in the project and its modules. SDK and graph probes +//! are cached until manifests or settings change; Node packages are read on refresh. //! //! Archives are listed as browsable roots (`!`, see `ide_vfs`): the //! JDK's `src.zip` and the Maven / Gradle dependency jars (source attachments resolved by JDT). use crate::exe; -use crate::exec::{query_in, which}; +use crate::exec::{query_in, query_in_cancellable, which}; +use ide_core::CancellationToken; use ide_core::settings::Settings; use serde::{Deserialize, Serialize}; use specta::Type; @@ -51,7 +52,6 @@ print(json.dumps([p.get(k) for k in ('stdlib','platstdlib','purelib','platlib')] const JAVA_MANIFESTS: &[&str] = &["pom.xml", "build.gradle", "build.gradle.kts", "settings.gradle", "settings.gradle.kts"]; -const NODE_MANIFESTS: &[&str] = &["package.json", "package-lock.json", "yarn.lock", "pnpm-lock.yaml", "bun.lockb"]; const CARGO_MANIFESTS: &[&str] = &["Cargo.toml", "Cargo.lock", "rust-toolchain", "rust-toolchain.toml"]; const GO_MANIFESTS: &[&str] = &["go.mod", "go.sum", "go.work"]; const RUBY_MANIFESTS: &[&str] = &["Gemfile", "Gemfile.lock", ".ruby-version", ".tool-versions", "mise.toml"]; @@ -66,20 +66,210 @@ pub struct BuildDependency { pub version: String, } -/// The project's library groups; empty when no SDK is configured. Runs the +/// The project's SDK and dependency library groups. Runs the /// toolchains (in parallel), so call it off the UI / command thread. /// `dependencies` (from the build model) are listed when their jars are in the local repositories. pub fn project_libraries(root: &Path, settings: &Settings, dependencies: &[BuildDependency]) -> Vec { - type Detect = fn(&Path, &Settings) -> Vec; - let detectors: [Detect; 7] = - [python_groups, jdk_groups, node_groups, rust_groups, go_groups, ruby_groups, project_structure_groups]; + project_libraries_stream(root, settings, dependencies, &CancellationToken::new(), &|_| {}) +} + +/// Publishes each ecosystem independently; a failed or slow graph cannot hide other libraries. +pub fn project_libraries_stream( + root: &Path, + settings: &Settings, + dependencies: &[BuildDependency], + cancel: &CancellationToken, + on_groups: &(dyn Fn(Vec) + Sync), +) -> Vec { + type Detect = fn(&Path, &Settings, &CancellationToken) -> Vec; + let detectors: [Detect; 5] = [python_groups, jdk_groups, node_groups, go_groups, ruby_groups]; + let roots = library_project_roots(root, cancel); + let settings = module_settings(root, settings); std::thread::scope(|s| { - let handles: Vec<_> = detectors.iter().map(|d| s.spawn(move || d(root, settings))).collect(); - let deps = s.spawn(|| dependency_groups(root, settings, dependencies)); - handles.into_iter().chain([deps]).flat_map(|h| h.join().unwrap_or_default()).collect() + let handles: Vec<_> = detectors + .iter() + .map(|d| { + let (roots, settings) = (&roots, &settings); + s.spawn(move || { + let mut groups = Vec::new(); + for dir in roots { + if cancel.is_cancelled() { + break; + } + let batch = d(dir, settings, cancel); + if !cancel.is_cancelled() && !batch.is_empty() { + on_groups(batch.clone()); + groups.extend(batch); + } + } + groups + }) + }) + .collect(); + let rust = s.spawn(|| { + let mut seen = std::collections::HashSet::new(); + let mut groups = Vec::new(); + for dir in &roots { + if cancel.is_cancelled() { + break; + } + if !dir.join("Cargo.toml").is_file() { + continue; + } + let bin = crate::apply::rust_bin(dir, &settings).map(|(bin, _)| bin); + let workspace = tool(bin, "cargo") + .and_then(|cargo| { + query_in_cancellable( + &cargo, + &["locate-project", "--workspace", "--message-format", "plain"], + dir, + &[], + QUERY_TIMEOUT, + cancel, + ) + }) + .map(PathBuf::from) + .filter(|manifest| manifest.is_file()) + .and_then(|manifest| manifest.parent().map(Path::to_path_buf)) + .unwrap_or_else(|| dir.clone()); + if !seen.insert(workspace.clone()) { + continue; + } + let batch = rust_groups(&workspace, &settings, cancel); + if !cancel.is_cancelled() && !batch.is_empty() { + on_groups(batch.clone()); + groups.extend(batch); + } + } + groups + }); + let deps = s.spawn(|| { + if cancel.is_cancelled() { + return Vec::new(); + } + let mut groups = dependency_groups(root, &settings, dependencies); + groups.extend(project_structure_groups(root, &settings)); + if !cancel.is_cancelled() && !groups.is_empty() { + on_groups(groups.clone()); + } + groups + }); + merge_groups( + handles + .into_iter() + .chain([rust, deps]) + .flat_map(|h| { + h.join().unwrap_or_else(|error| { + tracing::warn!(?error, "external libraries provider panicked"); + Vec::new() + }) + }) + .collect(), + ) }) } +fn library_project_roots(root: &Path, cancel: &CancellationToken) -> Vec { + use ide_core::project_roots::{ + SEARCH_DEPTH, discover_matching_cancellable, expand_members, node_workspace_members, + }; + let mut roots = discover_matching_cancellable(root, SEARCH_DEPTH, false, cancel, |dir| { + [ + "package.json", + "pnpm-workspace.yaml", + "Cargo.toml", + "go.mod", + "go.work", + "Gemfile", + ".ruby-version", + "pyproject.toml", + "setup.py", + "requirements.txt", + ".python-version", + ".tool-versions", + "mise.toml", + ] + .iter() + .chain(JAVA_MANIFESTS) + .any(|name| dir.join(name).is_file()) + || [".venv", "venv", ".pixi"].iter().any(|name| dir.join(name).is_dir()) + || dir == root + }); + let mut scanned = std::collections::HashSet::new(); + let mut index = 0; + while index < roots.len() && !cancel.is_cancelled() { + let dir = roots[index].clone(); + index += 1; + if !scanned.insert(dir.clone()) { + continue; + } + let mut members = node_workspace_members(&dir); + if let Some(manifest) = std::fs::read_to_string(dir.join("Cargo.toml")) + .ok() + .and_then(|text| toml::from_str::(&text).ok()) + { + let patterns: Vec = manifest + .get("workspace") + .and_then(|w| w.get("members")) + .and_then(toml::Value::as_array) + .into_iter() + .flatten() + .filter_map(toml::Value::as_str) + .map(str::to_owned) + .collect(); + members.extend(expand_members(&dir, &patterns, "Cargo.toml")); + } + if let Ok(work) = std::fs::read_to_string(dir.join("go.work")) { + members.extend(ide_run::go::work_modules(&work).into_iter().map(|path| dir.join(path))); + } + roots.extend( + members + .into_iter() + .map(|dir| ide_core::project_roots::resolve(&dir, Path::new(""))) + .filter(|dir| !scanned.contains(dir)), + ); + } + roots.sort(); + roots.dedup(); + roots +} + +fn module_settings(root: &Path, settings: &Settings) -> Settings { + let mut settings = settings.clone(); + let l = &mut settings.languages; + for value in [ + &mut l.python.interpreter, + &mut l.ruby.interpreter, + &mut l.node.interpreter, + &mut l.rust.toolchain, + &mut l.rust.stdlib, + &mut l.go.goroot, + &mut l.go.gopath, + &mut l.java.jdk, + ] { + if let Some(path) = crate::apply::setting_path(root, value) { + *value = path.to_string_lossy().into_owned(); + } + } + settings +} + +pub fn merge_groups(groups: Vec) -> Vec { + let mut merged: Vec = Vec::new(); + for group in groups { + if let Some(existing) = merged.iter_mut().find(|g| g.kind == group.kind && g.name == group.name) { + for root in group.roots { + if !existing.roots.iter().any(|r| r.path == root.path) { + existing.roots.push(root); + } + } + } else { + merged.push(group); + } + } + merged +} + // ------------------------------------------------------------------ cache /// Size and modification time of each manifest (`None`: missing). @@ -117,17 +307,42 @@ pub(crate) fn cached( manifests: &[PathBuf], detect: impl FnOnce() -> Vec, ) -> Vec { + cached_optional(root, ecosystem, key, manifests, || Some(detect())).unwrap_or_default() +} + +fn cached_cancellable( + root: &Path, + ecosystem: &'static str, + key: String, + manifests: &[PathBuf], + cancel: &CancellationToken, + detect: impl FnOnce() -> Vec, +) -> Vec { + cached_optional(root, ecosystem, key, manifests, || { + let groups = detect(); + (!cancel.is_cancelled()).then_some(groups) + }) + .unwrap_or_default() +} + +fn cached_optional( + root: &Path, + ecosystem: &'static str, + key: String, + manifests: &[PathBuf], + detect: impl FnOnce() -> Option>, +) -> Option> { let id = (root.to_path_buf(), ecosystem); let now = stamp(manifests); if let Some(c) = CACHE.lock().unwrap_or_else(|e| e.into_inner()).get(&id) && c.key == key && c.stamp == now { - return c.groups.clone(); + return Some(c.groups.clone()); } - let groups = detect(); + let groups = detect()?; CACHE.lock().unwrap_or_else(|e| e.into_inner()).insert(id, Cached { key, stamp: now, groups: groups.clone() }); - groups + Some(groups) } /// `names` below `root` when at least one of them exists (the project uses that ecosystem). @@ -147,11 +362,11 @@ fn tool(bin: Option, name: &str) -> Option { // ------------------------------------------------------------------ python -fn python_groups(root: &Path, s: &Settings) -> Vec { +fn python_groups(root: &Path, s: &Settings, cancel: &CancellationToken) -> Vec { let Some(python) = crate::apply::python_interpreter(root, s) else { return Vec::new() }; let mut files = vec![python.clone()]; files.extend(crate::venv_of(&python).map(|v| v.join("pyvenv.cfg"))); - cached(root, "python", python.display().to_string(), &files, || { + cached_cancellable(root, "python", python.display().to_string(), &files, cancel, || { python_libraries(&python, Some(root)).into_iter().collect() }) } @@ -199,13 +414,13 @@ pub(crate) fn named_roots(roots: Vec<(String, PathBuf)>) -> Vec { // ------------------------------------------------------------------ ruby -fn ruby_groups(root: &Path, s: &Settings) -> Vec { +fn ruby_groups(root: &Path, s: &Settings, cancel: &CancellationToken) -> Vec { let Some(files) = manifests(root, RUBY_MANIFESTS) else { return Vec::new() }; if !root.join("Gemfile").is_file() && !root.join(".ruby-version").is_file() { return Vec::new(); } let Some(ruby) = crate::apply::ruby_interpreter(root, s) else { return Vec::new() }; - cached(root, "ruby", ruby.display().to_string(), &files, || { + cached_cancellable(root, "ruby", ruby.display().to_string(), &files, cancel, || { let name = crate::ruby::ruby_sdk(&ruby, "system").map(|s| s.name).unwrap_or_else(|| "Ruby".into()); let (stdlib, gems) = crate::ruby::library_roots(&ruby, Some(root)); let mut out = Vec::new(); @@ -217,13 +432,13 @@ fn ruby_groups(root: &Path, s: &Settings) -> Vec { // ------------------------------------------------------------------ jdk -fn jdk_groups(root: &Path, s: &Settings) -> Vec { +fn jdk_groups(root: &Path, s: &Settings, cancel: &CancellationToken) -> Vec { let files = match manifests(root, JAVA_MANIFESTS) { Some(files) => files, None if !s.languages.java.jdk.trim().is_empty() || !s.project_structure.source_roots.is_empty() => Vec::new(), None => return Vec::new(), }; - cached(root, "jdk", s.languages.java.jdk.clone(), &files, || { + cached_cancellable(root, "jdk", s.languages.java.jdk.clone(), &files, cancel, || { let jdk = crate::apply::project_jdk(root, s).or_else(|| { crate::java::pick_jdk(&crate::java::installed_jdks(), crate::java::project_java_version(root)).cloned() }); @@ -274,7 +489,7 @@ pub(crate) fn jdk_name(home: &Path) -> Option { // ------------------------------------------------------------------ maven / gradle -fn dependency_groups(root: &Path, s: &Settings, deps: &[BuildDependency]) -> Vec { +pub fn dependency_groups(root: &Path, s: &Settings, deps: &[BuildDependency]) -> Vec { if deps.is_empty() { return Vec::new(); } @@ -388,24 +603,27 @@ pub(crate) fn gradle_jar(home: &Path, d: &BuildDependency) -> Option { // ------------------------------------------------------------------ node -fn node_groups(root: &Path, s: &Settings) -> Vec { - let Some(files) = manifests(root, NODE_MANIFESTS) else { return Vec::new() }; +fn node_groups(root: &Path, s: &Settings, _cancel: &CancellationToken) -> Vec { if !root.join("package.json").is_file() { return Vec::new(); } - cached(root, "node", s.languages.node.interpreter.clone(), &files, || { - let node = tool(crate::apply::node_dir(root, s), "node"); - node_libraries(root, node.as_deref()) - }) + let node = tool(crate::apply::node_dir(root, s), "node"); + node_libraries(root, node.as_deref()) } /// `node_modules/` of the direct dependencies in `package.json`, and the /// Node.js core typings (`@types/node`, from the project or the runtime's global `lib`). pub(crate) fn node_libraries(root: &Path, node: Option<&Path>) -> Vec { - let manifest = std::fs::read_to_string(root.join("package.json")).unwrap_or_default(); - let modules = root.join("node_modules"); - let deps = named_roots(package_dependencies(&manifest).into_iter().map(|d| (d.clone(), modules.join(d))).collect()); - let mut core = vec![modules.join("@types/node")]; + let manifest = std::fs::read_to_string(root.join("package.json")).unwrap_or_else(|error| { + tracing::warn!(?root, %error, "cannot read package.json for external libraries"); + String::new() + }); + let resolve = + |name: &str| root.ancestors().map(|dir| dir.join("node_modules").join(name)).find(|path| path.is_dir()); + let deps = named_roots( + package_dependencies(&manifest).into_iter().filter_map(|d| Some((d.clone(), resolve(&d)?))).collect(), + ); + let mut core: Vec<_> = resolve("@types/node").into_iter().collect(); // `/bin/node` → `/lib/node_modules`. if let Some(prefix) = node.and_then(|n| dunce::canonicalize(n).ok()).and_then(|n| Some(n.parent()?.parent()?.to_path_buf())) @@ -422,7 +640,13 @@ pub(crate) fn node_libraries(root: &Path, node: Option<&Path>) -> Vec Vec { - let Ok(json) = serde_json::from_str::(package_json) else { return Vec::new() }; + let json = match serde_json::from_str::(package_json) { + Ok(json) => json, + Err(error) => { + tracing::warn!(%error, "cannot parse package.json for external libraries"); + return Vec::new(); + } + }; let mut names: Vec = ["dependencies", "devDependencies", "optionalDependencies", "peerDependencies"] .iter() .filter_map(|k| json.get(k)?.as_object()) @@ -436,19 +660,19 @@ pub(crate) fn package_dependencies(package_json: &str) -> Vec { // ------------------------------------------------------------------ rust -fn rust_groups(root: &Path, s: &Settings) -> Vec { +fn rust_groups(root: &Path, s: &Settings, cancel: &CancellationToken) -> Vec { let Some(files) = manifests(root, CARGO_MANIFESTS) else { return Vec::new() }; if !root.join("Cargo.toml").is_file() { return Vec::new(); } let r = &s.languages.rust; - cached(root, "rust", format!("{}\n{}", r.toolchain, r.stdlib), &files, || { - let bin = crate::apply::rust_bin(root, s).map(|(b, _)| b); + let bin = crate::apply::rust_bin(root, s).map(|(b, _)| b); + let mut out = cached_cancellable(root, "rust", format!("{}\n{}", r.toolchain, r.stdlib), &files, cancel, || { let mut out = Vec::new(); if let Some(rustc) = tool(bin.clone(), "rustc") { - let version = query_in(&rustc, &["--version"], Some(root), &[], QUERY_TIMEOUT); + let version = query_in_cancellable(&rustc, &["--version"], root, &[], QUERY_TIMEOUT, cancel); let library = crate::apply::setting_path(root, &r.stdlib).or_else(|| { - let sysroot = query_in(&rustc, &["--print", "sysroot"], Some(root), &[], QUERY_TIMEOUT)?; + let sysroot = query_in_cancellable(&rustc, &["--print", "sysroot"], root, &[], QUERY_TIMEOUT, cancel)?; Some(PathBuf::from(sysroot).join("lib/rustlib/src/rust/library")) }); let name = match version.as_deref().and_then(|v| v.split_whitespace().nth(1)) { @@ -457,43 +681,104 @@ fn rust_groups(root: &Path, s: &Settings) -> Vec { }; out.extend(library.and_then(|l| group(name, "rust", library_roots(vec![l])))); } - if let Some(cargo) = tool(bin, "cargo") - && let Some(json) = - query_in(&cargo, &["metadata", "--format-version", "1", "--offline"], Some(root), &[], GRAPH_TIMEOUT) - { - out.extend(group("Cargo dependencies", "cargo", named_roots(parse_cargo_metadata(&json)))); - } out - }) + }); + if let Some(cargo) = tool(bin, "cargo") { + let graph = cached_optional(root, "cargo", r.toolchain.clone(), &files, || { + let json = query_in_cancellable( + &cargo, + &["metadata", "--format-version", "1", "--offline"], + root, + &[], + GRAPH_TIMEOUT, + cancel, + )?; + let roots = parse_cargo_metadata(&json)?; + Some(group("Cargo dependencies", "cargo", named_roots(roots)).into_iter().collect()) + }); + out.extend(graph.unwrap_or_else(|| { + let home = ide_core::process::host_var_os("CARGO_HOME") + .map(PathBuf::from) + .or_else(|| Some(crate::home_dir()?.join(".cargo"))); + let roots = home.map(|home| cargo_lock_roots(root, &root.join(home))).unwrap_or_default(); + group("Cargo dependencies", "cargo", named_roots(roots)).into_iter().collect() + })); + } + out +} + +/// An incomplete offline graph still has useful downloaded sources recorded in Cargo.lock. +fn cargo_lock_roots(root: &Path, home: &Path) -> Vec<(String, PathBuf)> { + let lock = std::fs::read_to_string(root.join("Cargo.lock")).ok(); + let Some(lock) = lock.and_then(|s| toml::from_str::(&s).ok()) else { + return Vec::new(); + }; + let Some(packages) = lock.get("package").and_then(toml::Value::as_array) else { + return Vec::new(); + }; + let sources: Vec = + std::fs::read_dir(home.join("registry/src")).into_iter().flatten().flatten().map(|e| e.path()).collect(); + packages + .iter() + .filter_map(|p| { + let source = p.get("source")?.as_str()?; + // Other registries cannot be identified safely from a hashed cache folder alone. + if source != "registry+https://github.com/rust-lang/crates.io-index" { + return None; + } + let name = p.get("name")?.as_str()?; + let version = p.get("version")?.as_str()?; + let path = sources + .iter() + .filter(|dir| { + dir.file_name().is_some_and(|n| { + let n = n.to_string_lossy(); + n.starts_with("index.crates.io-") || n.starts_with("github.com-") + }) + }) + .map(|dir| dir.join(format!("{name}-{version}"))) + .find(|dir| dir.join("Cargo.toml").is_file())?; + Some((format!("{name} {version}"), path)) + }) + .collect() } /// `(name version, source dir)` of the non-workspace packages in `cargo metadata` output, by name. -pub(crate) fn parse_cargo_metadata(json: &str) -> Vec<(String, PathBuf)> { +pub(crate) fn parse_cargo_metadata(json: &str) -> Option> { #[derive(Deserialize)] struct Package { name: String, version: String, + id: Option, source: Option, manifest_path: PathBuf, } #[derive(Deserialize)] struct Metadata { packages: Vec, + #[serde(default)] + workspace_members: Vec, } - let Ok(meta) = serde_json::from_str::(json) else { return Vec::new() }; + let meta = match serde_json::from_str::(json) { + Ok(meta) => meta, + Err(error) => { + tracing::warn!(%error, "cannot parse Cargo metadata for external libraries"); + return None; + } + }; let mut out: Vec<(String, PathBuf)> = meta .packages .into_iter() - .filter(|p| p.source.is_some()) + .filter(|p| p.source.is_some() || p.id.as_ref().is_some_and(|id| !meta.workspace_members.contains(id))) .filter_map(|p| Some((format!("{} {}", p.name, p.version), p.manifest_path.parent()?.to_path_buf()))) .collect(); out.sort_by_cached_key(|(name, _)| name.to_lowercase()); - out + Some(out) } // ------------------------------------------------------------------ go -fn go_groups(root: &Path, s: &Settings) -> Vec { +fn go_groups(root: &Path, s: &Settings, cancel: &CancellationToken) -> Vec { let Some(mut files) = manifests(root, GO_MANIFESTS) else { return Vec::new() }; if !root.join("go.mod").is_file() && !root.join("go.work").is_file() { return Vec::new(); @@ -505,7 +790,7 @@ fn go_groups(root: &Path, s: &Settings) -> Vec { } files.extend([root.join("go.work.sum"), root.join("vendor/modules.txt")]); let g = &s.languages.go; - cached(root, "go", serde_json::to_string(g).unwrap_or_default(), &files, || { + cached_cancellable(root, "go", serde_json::to_string(g).unwrap_or_default(), &files, cancel, || { let goroot = crate::apply::goroot(root, s); let Some(go) = tool(goroot.as_ref().map(|r| r.join("bin")), "go") else { return Vec::new() }; let goroot_s = goroot.as_ref().map(|p| p.to_string_lossy().into_owned()); @@ -518,7 +803,9 @@ fn go_groups(root: &Path, s: &Settings) -> Vec { env.extend(goroot_s.as_deref().map(|v| ("GOROOT", v))); env.extend(gopath.as_deref().map(|v| ("GOPATH", v))); let mut out = Vec::new(); - if let Some(vars) = query_in(&go, &["env", "GOROOT", "GOVERSION"], Some(root), &env, QUERY_TIMEOUT) { + if let Some(vars) = + query_in_cancellable(&go, &["env", "GOROOT", "GOVERSION"], root, &env, QUERY_TIMEOUT, cancel) + { let mut lines = vars.lines().map(str::trim); let (dir, version) = (lines.next().unwrap_or_default(), lines.next().unwrap_or_default()); let name = match version.strip_prefix("go") { @@ -539,7 +826,8 @@ fn go_groups(root: &Path, s: &Settings) -> Vec { let modules = vendor_roots(&vendor, &text); out.extend(group("Go Modules", "goModules", named_roots(modules))); } - } else if let Some(text) = query_in(&go, &["list", "-m", "-e", "-json", "all"], Some(root), &env, GRAPH_TIMEOUT) + } else if let Some(text) = + query_in_cancellable(&go, &["list", "-m", "-e", "-json", "all"], root, &env, GRAPH_TIMEOUT, cancel) { out.extend(group("Go Modules", "goModules", named_roots(parse_go_modules(&text)))); } @@ -590,6 +878,191 @@ pub(crate) fn parse_go_modules(text: &str) -> Vec<(String, PathBuf)> { mod go_cache_tests { use super::*; + #[test] + fn library_discovery_includes_declared_workspace_members_beyond_the_search_depth() { + let tmp = tempfile::tempdir().unwrap(); + let root = tmp.path(); + std::fs::write(root.join("package.json"), r#"{"workspaces":["nested/very/deep/packages/*"]}"#).unwrap(); + std::fs::write(root.join("Cargo.toml"), "[workspace]\nmembers=['nested/very/deep/crates/*']\n").unwrap(); + std::fs::write(root.join("go.work"), "go 1.22\nuse ./nested/very/deep/go\n").unwrap(); + for (dir, marker) in [("packages/web", "package.json"), ("crates/api", "Cargo.toml"), ("go", "go.mod")] { + let member = root.join("nested/very/deep").join(dir); + std::fs::create_dir_all(&member).unwrap(); + std::fs::write(member.join(marker), "").unwrap(); + } + let roots = library_project_roots(root, &CancellationToken::new()); + for member in ["packages/web", "crates/api", "go"] { + assert!(roots.contains(&root.join("nested/very/deep").join(member)), "{member}"); + } + } + + #[cfg(unix)] + #[test] + fn cancelled_sdk_queries_do_not_poison_the_next_refresh() { + use std::os::unix::fs::PermissionsExt; + use std::time::Instant; + let tmp = tempfile::tempdir().unwrap(); + let root = tmp.path(); + std::fs::write(root.join("Cargo.toml"), "[package]\nname='app'\nversion='0.1.0'\n").unwrap(); + let bin = root.join("tools"); + std::fs::create_dir(&bin).unwrap(); + let library = root.join("stdlib"); + std::fs::create_dir(&library).unwrap(); + let started = root.join("rustc-started"); + for (name, script) in [ + ("rustc", format!("#!/bin/sh\ntouch '{}'\nexec sleep 60\n", started.display())), + ("cargo", "#!/bin/sh\nexit 1\n".into()), + ] { + let path = bin.join(name); + std::fs::write(&path, script).unwrap(); + std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o755)).unwrap(); + } + let mut settings = Settings::default(); + settings.languages.rust.toolchain = "tools".into(); + settings.languages.rust.stdlib = "stdlib".into(); + let cancel = CancellationToken::new(); + std::thread::scope(|scope| { + let task = scope.spawn(|| project_libraries_stream(root, &settings, &[], &cancel, &|_| {})); + let deadline = Instant::now() + Duration::from_secs(5); + while !started.is_file() && Instant::now() < deadline { + std::thread::sleep(Duration::from_millis(5)); + } + cancel.cancel(); + task.join().unwrap(); + assert!(started.is_file()); + }); + std::fs::write(bin.join("rustc"), "#!/bin/sh\necho 'rustc 1.2.3'\n").unwrap(); + let groups = project_libraries(root, &settings, &[]); + assert_eq!(groups.len(), 1); + assert_eq!(groups[0].name, "Rust 1.2.3"); + assert_eq!(groups[0].roots[0].path, library); + } + + #[test] + fn libraries_keep_downloaded_crates_when_an_offline_graph_is_incomplete() { + let tmp = tempfile::tempdir().unwrap(); + let root = tmp.path().join("project"); + let home = tmp.path().join("cargo"); + std::fs::create_dir_all(&root).unwrap(); + let serde = home.join("registry/src/index.crates.io-test/serde-1.0.200"); + std::fs::create_dir_all(&serde).unwrap(); + std::fs::write(serde.join("Cargo.toml"), "").unwrap(); + std::fs::write( + root.join("Cargo.lock"), + r#" + version = 4 + [[package]] + name = "serde" + version = "1.0.200" + source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] + name = "not-downloaded" + version = "9.0.0" + source = "registry+https://github.com/rust-lang/crates.io-index" + "#, + ) + .unwrap(); + assert_eq!(cargo_lock_roots(&root, &home), [("serde 1.0.200".into(), serde)]); + } + + #[cfg(unix)] + #[test] + fn failed_cargo_queries_are_retried_without_manifest_changes() { + use std::os::unix::fs::PermissionsExt; + let tmp = tempfile::tempdir().unwrap(); + let root = tmp.path(); + std::fs::write(root.join("Cargo.toml"), "[workspace]\nmembers=['members/*']\n").unwrap(); + let member = root.join("members/app"); + std::fs::create_dir_all(&member).unwrap(); + std::fs::write(member.join("Cargo.toml"), "[package]\nname='app'\nversion='0.1.0'\n").unwrap(); + let bin = root.join("tools"); + std::fs::create_dir(&bin).unwrap(); + for name in ["cargo", "rustc"] { + let path = bin.join(name); + std::fs::write(&path, "#!/bin/sh\nexit 1\n").unwrap(); + std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o755)).unwrap(); + } + let mut settings = Settings::default(); + settings.languages.rust.toolchain = "tools".into(); + assert!(project_libraries(root, &settings, &[]).is_empty()); + let source = root.join("local"); + std::fs::create_dir(&source).unwrap(); + let json = serde_json::json!({"packages": [{"id":"dep", "name":"dep", "version":"1.0.0", "source":null, + "manifest_path":source.join("Cargo.toml")}], "workspace_members":[]}); + let calls = root.join("metadata-calls"); + std::fs::write(bin.join("cargo"), format!("#!/bin/sh\nif [ \"$1\" = locate-project ]; then echo '{}'; exit; fi\necho called >> '{}'\ncat <<'JSON'\n{json}\nJSON\n", root.join("Cargo.toml").display(), calls.display())).unwrap(); + let groups = project_libraries(root, &settings, &[]); + assert_eq!(groups.len(), 1); + assert_eq!(groups[0].kind, "cargo"); + assert_eq!(groups[0].roots[0].path, source); + assert_eq!(std::fs::read_to_string(calls).unwrap().lines().count(), 1); + } + + #[cfg(unix)] + #[test] + fn libraries_publish_node_and_maven_while_cargo_is_running_and_cancel_it() { + use std::os::unix::fs::PermissionsExt; + use std::sync::mpsc; + use std::time::Instant; + + let tmp = tempfile::tempdir().unwrap(); + let root = tmp.path(); + std::fs::write(root.join("Cargo.toml"), "[package]\nname='app'\nversion='0.1.0'\n").unwrap(); + std::fs::write(root.join("package.json"), r#"{"dependencies":{"dep":"1"}}"#).unwrap(); + std::fs::create_dir_all(root.join("node_modules/dep")).unwrap(); + let bin = root.join("tools"); + std::fs::create_dir(&bin).unwrap(); + let started = root.join("cargo-started"); + for (name, script) in [ + ("rustc", "#!/bin/sh\nexit 1\n".to_owned()), + ( + "cargo", + format!( + "#!/bin/sh\nif [ \"$1\" = locate-project ]; then echo '{}'; exit; fi\ntouch '{}'\nexec sleep 60\n", + root.join("Cargo.toml").display(), + started.display() + ), + ), + ] { + let path = bin.join(name); + std::fs::write(&path, script).unwrap(); + std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o755)).unwrap(); + } + let mut settings = Settings::default(); + settings.languages.rust.toolchain = "tools".into(); + settings.languages.java.maven.local_repository = root.join("m2").to_string_lossy().into_owned(); + let jar = root.join("m2/g/a/1/a-1.jar"); + std::fs::create_dir_all(jar.parent().unwrap()).unwrap(); + std::fs::write(jar, "").unwrap(); + let deps = [BuildDependency { gradle: false, group: "g".into(), artifact: "a".into(), version: "1".into() }]; + let cancel = CancellationToken::new(); + let (tx, rx) = mpsc::channel(); + std::thread::scope(|scope| { + let task = scope.spawn(|| { + project_libraries_stream(root, &settings, &deps, &cancel, &|g| { + tx.send(g).unwrap(); + }) + }); + let deadline = Instant::now() + Duration::from_secs(5); + let mut kinds = Vec::new(); + while Instant::now() < deadline + && !(kinds.contains(&"nodeModules".to_owned()) + && kinds.contains(&"maven".to_owned()) + && started.is_file()) + { + if let Ok(groups) = rx.recv_timeout(Duration::from_millis(20)) { + kinds.extend(groups.into_iter().map(|g| g.kind)); + } + } + cancel.cancel(); + task.join().unwrap(); + assert!(started.is_file(), "cargo query should have started"); + assert!(kinds.contains(&"nodeModules".to_owned())); + assert!(kinds.contains(&"maven".to_owned())); + assert!(Instant::now() < deadline, "cancellation should stop the graph query promptly"); + }); + } + #[test] fn finds_published_version_catalogs_in_the_gradle_cache_and_maven_repository() { let tmp = tempfile::tempdir().unwrap(); diff --git a/crates/ide-sdk/src/tests.rs b/crates/ide-sdk/src/tests.rs index a84942a7b..f6d37b023 100644 --- a/crates/ide-sdk/src/tests.rs +++ b/crates/ide-sdk/src/tests.rs @@ -1006,6 +1006,13 @@ fn node_libraries_list_direct_dependencies_and_core_typings() { for dir in ["react", "@scope/ui", "@types/node", "transitive"] { std::fs::create_dir_all(root.join("node_modules").join(dir)).unwrap(); } + #[cfg(unix)] + { + let package = root.join("node_modules/.pnpm/@scope+ui@1/node_modules/@scope/ui"); + std::fs::create_dir_all(&package).unwrap(); + std::fs::remove_dir(root.join("node_modules/@scope/ui")).unwrap(); + std::os::unix::fs::symlink(package, root.join("node_modules/@scope/ui")).unwrap(); + } let groups = crate::libraries::node_libraries(root, None); let summary: Vec<(&str, &str, Vec<&str>)> = groups .iter() @@ -1022,16 +1029,46 @@ fn node_libraries_list_direct_dependencies_and_core_typings() { } #[test] -fn cargo_metadata_lists_registry_packages_only() { +fn libraries_include_nested_packages_and_refresh_after_install() { + let tmp = tempfile::tempdir().unwrap(); + let root = tmp.path(); + for dir in ["", "packages/web", "packages/api", "node_modules/ignored", "target/ignored"] { + let package = root.join(dir); + std::fs::create_dir_all(&package).unwrap(); + std::fs::write(package.join("package.json"), r#"{"dependencies":{"dep":"1"}}"#).unwrap(); + } + let settings = Settings::default(); + assert!(crate::libraries::project_libraries(root, &settings, &[]).is_empty()); + for dir in ["packages/web", "packages/api", "node_modules/ignored", "target/ignored"] { + std::fs::create_dir_all(root.join(dir).join("node_modules/dep")).unwrap(); + } + let groups = crate::libraries::project_libraries(root, &settings, &[]); + let node = groups.iter().find(|g| g.kind == "nodeModules").expect("nested packages after install"); + assert_eq!(node.roots.len(), 2); + assert!(node.roots.iter().all(|r| r.path.starts_with(root.join("packages")))); + std::fs::remove_dir(root.join("packages/web/node_modules/dep")).unwrap(); + std::fs::create_dir_all(root.join("node_modules/dep")).unwrap(); + let groups = crate::libraries::project_libraries(root, &settings, &[]); + let node = groups.iter().find(|g| g.kind == "nodeModules").unwrap(); + assert_eq!(node.roots.len(), 2, "hoisted dependencies are shared across members"); +} + +#[test] +fn cargo_metadata_lists_registry_and_non_workspace_path_dependencies() { let tmp = tempfile::tempdir().unwrap(); let serde = tmp.path().join("registry/src/index/serde-1.0.200"); let json = serde_json::json!({ "packages": [ - { "name": "app", "version": "0.1.0", "source": null, "manifest_path": tmp.path().join("Cargo.toml") }, + { "name": "app", "id": "app", "version": "0.1.0", "source": null, "manifest_path": tmp.path().join("Cargo.toml") }, { "name": "serde", "version": "1.0.200", "source": "registry+https://github.com/rust-lang/crates.io-index", "manifest_path": serde.join("Cargo.toml") }, - ]}); - assert_eq!(crate::libraries::parse_cargo_metadata(&json.to_string()), [("serde 1.0.200".to_owned(), serde)]); - assert!(crate::libraries::parse_cargo_metadata("error: no Cargo.toml").is_empty()); + { "name": "local", "id": "local", "version": "0.2.0", "source": null, + "manifest_path": tmp.path().join("local/Cargo.toml") }, + ], "workspace_members": ["app"]}); + assert_eq!( + crate::libraries::parse_cargo_metadata(&json.to_string()).unwrap(), + [("local 0.2.0".to_owned(), tmp.path().join("local")), ("serde 1.0.200".to_owned(), serde),] + ); + assert!(crate::libraries::parse_cargo_metadata("error: no Cargo.toml").is_none()); } #[test] diff --git a/src-tauri/src/commands/sdk.rs b/src-tauri/src/commands/sdk.rs index a8053f8c4..a925f2cc8 100644 --- a/src-tauri/src/commands/sdk.rs +++ b/src-tauri/src/commands/sdk.rs @@ -77,8 +77,11 @@ pub async fn sdk_php_composer_install( pub async fn sdk_project_libraries( window: Window, state: State<'_, AppState>, + request: RequestId, + on_groups: Channel>, ) -> IdeResult> { - ide_host::ops::sdk::sdk_project_libraries(&state.host, window.label()).await + ide_host::ops::sdk::sdk_project_libraries(&state.host, window.label(), request, crate::commands::sender(on_groups)) + .await } /// Looks up `names` (bare command names): `node_modules/.bin` above `near`, then the search path. diff --git a/src/bindings.ts b/src/bindings.ts index f57c5eccf..3afdf3b0c 100644 --- a/src/bindings.ts +++ b/src/bindings.ts @@ -841,7 +841,7 @@ export const commands = { * SDK / library source folders shown under "External Libraries" in the Project view, * with the Maven / Gradle dependency jars of the build model. */ - sdkProjectLibraries: () => typedError(__TAURI_INVOKE("sdk_project_libraries")), + sdkProjectLibraries: (request: RequestId, onGroups: Channel) => typedError(__TAURI_INVOKE("sdk_project_libraries", { request, onGroups })), /** Looks up `names` (bare command names): `node_modules/.bin` above `near`, then the search path. */ sdkLocateTools: (names: string[], near: string | null) => typedError(__TAURI_INVOKE("sdk_locate_tools", { names, near })), /** Creates a Python environment with `tool` (`venv` / `uv` / `poetry`); output streams through `on_output`. */ diff --git a/src/features/libraries/index.ts b/src/features/libraries/index.ts index b18c03099..0f7d7c52e 100644 --- a/src/features/libraries/index.ts +++ b/src/features/libraries/index.ts @@ -6,10 +6,12 @@ * read-only, like IDEA's library sources. Archives (the JDK's `src.zip`, jars) * are browsed through the backend's `!/` paths. */ -import { commands, type LibraryGroup } from "@/bindings"; +import { Channel } from "@tauri-apps/api/core"; +import { commands, type LibraryGroup, type RequestId } from "@/bindings"; +import { mergeLibraryGroups } from "@/platform/libraries"; import { registerActions } from "@/platform/actions"; import { setLibraryRoots } from "@/editor/api"; -import { call, isTauri, onIdeEvent } from "@/platform/ipc"; +import { call, isCancelled, isTauri, nextRequestId, onIdeEvent } from "@/platform/ipc"; import { useProject } from "@/platform/project"; import { registerProjectViewRoot, type ProjectViewRootContribution } from "@/workbench/contributions"; @@ -45,6 +47,8 @@ const MANIFESTS = new Set([ "package-lock.json", "yarn.lock", "pnpm-lock.yaml", + "pnpm-workspace.yaml", + "bun.lock", "bun.lockb", "Cargo.toml", "Cargo.lock", @@ -53,6 +57,15 @@ const MANIFESTS = new Set([ "go.mod", "go.sum", "go.work", + "pyproject.toml", + "setup.py", + "requirements.txt", + ".python-version", + "Gemfile", + "Gemfile.lock", + ".ruby-version", + ".tool-versions", + "mise.toml", ]); /** Whether an `fsChanged` event touches a library manifest. */ @@ -94,6 +107,7 @@ const sdkKey = (s: ReturnType) => { s.project?.root ?? null, s.project?.trusted, l?.python?.interpreter, + l?.ruby?.interpreter, l?.java?.jdk, l?.node?.interpreter, l?.rust?.toolchain, @@ -102,6 +116,7 @@ const sdkKey = (s: ReturnType) => { l?.go?.gopath, l?.go?.modules, s.settings?.projectStructure?.libraries, + s.settings?.projectStructure?.sourceRoots, ]); }; @@ -112,6 +127,7 @@ export function activate(): () => void { let disposeReadOnly = () => {}; let seq = 0; let shown = ""; + let request: RequestId | undefined; const clear = () => { disposeRoot(); disposeReadOnly(); @@ -120,24 +136,42 @@ export function activate(): () => void { }; const refresh = () => { const id = ++seq; + if (request !== undefined) void commands.cancelRequest(request).catch(() => {}); + request = undefined; if (!useProject.getState().project || useProject.getState().project?.trusted === false || !isTauri()) return clear(); - call(commands.sdkProjectLibraries()) + let collected: LibraryGroup[] = []; + const show = (groups: LibraryGroup[]) => { + if (id !== seq) return; + const key = JSON.stringify(groups); + if (key === shown) return; + clear(); + shown = key; + const root = librariesRoot(groups); + if (!root) return; + disposeReadOnly = setLibraryRoots( + "externalLibraries", + groups.flatMap((g) => g.roots.map((r) => r.path)), + ); + disposeRoot = registerProjectViewRoot(root); + }; + const channel = new Channel(); + channel.onmessage = (groups) => { + collected = mergeLibraryGroups([...collected, ...groups]); + show(collected); + }; + const current = (request = nextRequestId()); + call(commands.sdkProjectLibraries(current, channel)) .then((groups) => { - if (id !== seq) return; - const key = JSON.stringify(groups); - if (key === shown) return; - clear(); - shown = key; - const root = librariesRoot(groups); - if (!root) return; - disposeReadOnly = setLibraryRoots( - "externalLibraries", - groups.flatMap((g) => g.roots.map((r) => r.path)), - ); - disposeRoot = registerProjectViewRoot(root); + collected = groups; + show(groups); }) - .catch((e: unknown) => console.warn("external libraries: cannot list SDK roots", e)); + .catch((e: unknown) => { + if (!isCancelled(e)) console.warn("external libraries: cannot list SDK roots", e); + }) + .finally(() => { + if (request === current) request = undefined; + }); }; refresh(); const disposeActions = registerActions([ @@ -154,6 +188,7 @@ export function activate(): () => void { }); return () => { seq++; + if (request !== undefined) void commands.cancelRequest(request).catch(() => {}); clearTimeout(timer); unsub(); unsubFs(); diff --git a/src/features/libraries/libraries.test.ts b/src/features/libraries/libraries.test.ts index 7d64dc1b2..86a41eaf6 100644 --- a/src/features/libraries/libraries.test.ts +++ b/src/features/libraries/libraries.test.ts @@ -1,6 +1,7 @@ import { afterEach, describe, expect, it } from "vitest"; import { isLibraryPath, setLibraryRoots } from "@/editor/api"; import { librariesRoot, touchesManifest } from "./index"; +import { mergeLibraryGroups } from "@/platform/libraries"; describe("External Libraries", () => { let dispose = () => {}; @@ -72,6 +73,7 @@ describe("External Libraries", () => { expect(touchesManifest(["/p/src/main.rs", "/p/Cargo.toml"])).toBe(true); expect(touchesManifest(["C:\\p\\go.mod"])).toBe(true); expect(touchesManifest(["/p/.venv/pyvenv.cfg"])).toBe(true); + expect(touchesManifest(["/p/pnpm-workspace.yaml", "/p/Gemfile.lock"])).toBe(true); expect(touchesManifest(["/p/node_modules/react/package.json"])).toBe(false); expect(touchesManifest(["/p/src/package.json.bak", "/p/README.md"])).toBe(false); }); @@ -83,4 +85,28 @@ describe("External Libraries", () => { dispose(); expect(isLibraryPath("/py/lib/python3.12/os.py")).toBe(false); }); + + it("merges streamed module groups without dropping ecosystems or distinct roots", () => { + const group = (kind: string, path: string) => ({ name: kind, kind, roots: [{ name: "dep", path }] }); + expect( + mergeLibraryGroups([ + group("cargo", "/cargo/serde"), + group("nodeModules", "/web/node_modules/dep"), + group("maven", "/m2/dep.jar!"), + group("nodeModules", "/api/node_modules/dep"), + group("cargo", "/cargo/serde"), + ]), + ).toEqual([ + group("cargo", "/cargo/serde"), + { + name: "nodeModules", + kind: "nodeModules", + roots: [ + { name: "dep", path: "/web/node_modules/dep" }, + { name: "dep", path: "/api/node_modules/dep" }, + ], + }, + group("maven", "/m2/dep.jar!"), + ]); + }); }); diff --git a/src/features/settings/languages/ProjectStructurePage.tsx b/src/features/settings/languages/ProjectStructurePage.tsx index 8ede36888..db7a8ec87 100644 --- a/src/features/settings/languages/ProjectStructurePage.tsx +++ b/src/features/settings/languages/ProjectStructurePage.tsx @@ -5,6 +5,7 @@ */ import { useCallback, useEffect, useId, useMemo, useState } from "react"; import clsx from "clsx"; +import { Channel } from "@tauri-apps/api/core"; import { Button, Checkbox, @@ -22,7 +23,8 @@ import { T, } from "@/ui"; import { SettingsFormContext } from "@/workbench/contributions"; -import { call, commands, isTauri, type FileEntry } from "@/platform/ipc"; +import { call, commands, isTauri, nextRequestId, type FileEntry } from "@/platform/ipc"; +import { mergeLibraryGroups } from "@/platform/libraries"; import type { BuildModel, LibraryGroup, WebArtifact } from "@/bindings"; import { relativePath, useProject } from "@/platform/project"; import { PageContext, useSettingsFormFor } from "../components/context"; @@ -988,11 +990,22 @@ function LibrariesSection() { useEffect(() => { if (!root || !isTauri()) return; let live = true; - call(commands.sdkProjectLibraries()) - .then((groups) => live && setDetectedGroups(groups)) + const request = nextRequestId(); + const channel = new Channel(); + let collected: LibraryGroup[] = []; + channel.onmessage = (groups) => { + collected = mergeLibraryGroups([...collected, ...groups]); + if (live) setDetectedGroups(collected); + }; + call(commands.sdkProjectLibraries(request, channel)) + .then((groups) => { + collected = groups; + if (live) setDetectedGroups(groups); + }) .catch(() => {}); return () => { live = false; + void commands.cancelRequest(request).catch(() => {}); }; }, [root, libs.length]); diff --git a/src/platform/libraries.ts b/src/platform/libraries.ts new file mode 100644 index 000000000..be60047ae --- /dev/null +++ b/src/platform/libraries.ts @@ -0,0 +1,17 @@ +import type { LibraryGroup } from "@/bindings"; + +/** Merge shared SDKs and package groups without losing different module roots. */ +export function mergeLibraryGroups(groups: readonly LibraryGroup[]): LibraryGroup[] { + const merged: LibraryGroup[] = []; + for (const group of groups) { + const existing = merged.find((g) => g.kind === group.kind && g.name === group.name); + if (!existing) { + merged.push({ ...group, roots: [...group.roots] }); + continue; + } + for (const root of group.roots) { + if (!existing.roots.some((r) => r.path === root.path)) existing.roots.push(root); + } + } + return merged; +}