Unnamed repository; edit this file 'description' to name the repository.
Auto merge of #15205 - Veykril:load-cargo, r=Veykril
Split out project loading capabilities from rust-analyzer crate
External tools currently depend on the entire lsp infra for no good reason so let's lift that out so those tools have something better to depend on
| -rw-r--r-- | Cargo.lock | 19 | ||||
| -rw-r--r-- | Cargo.toml | 1 | ||||
| -rw-r--r-- | crates/load-cargo/Cargo.toml | 23 | ||||
| -rw-r--r-- | crates/load-cargo/src/lib.rs | 441 | ||||
| -rw-r--r-- | crates/rust-analyzer/Cargo.toml | 2 | ||||
| -rw-r--r-- | crates/rust-analyzer/src/cli.rs | 1 | ||||
| -rw-r--r-- | crates/rust-analyzer/src/cli/analysis_stats.rs | 5 | ||||
| -rw-r--r-- | crates/rust-analyzer/src/cli/diagnostics.rs | 6 | ||||
| -rw-r--r-- | crates/rust-analyzer/src/cli/load_cargo.rs | 204 | ||||
| -rw-r--r-- | crates/rust-analyzer/src/cli/lsif.rs | 22 | ||||
| -rw-r--r-- | crates/rust-analyzer/src/cli/run_tests.rs | 9 | ||||
| -rw-r--r-- | crates/rust-analyzer/src/cli/scip.rs | 13 | ||||
| -rw-r--r-- | crates/rust-analyzer/src/cli/ssr.rs | 6 | ||||
| -rw-r--r-- | crates/rust-analyzer/src/global_state.rs | 3 | ||||
| -rw-r--r-- | crates/rust-analyzer/src/integrated_benchmarks.rs | 2 | ||||
| -rw-r--r-- | crates/rust-analyzer/src/reload.rs | 265 |
16 files changed, 518 insertions, 504 deletions
diff --git a/Cargo.lock b/Cargo.lock index 1f191cf794..e94dd9b407 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -963,6 +963,23 @@ dependencies = [ ] [[package]] +name = "load-cargo" +version = "0.0.0" +dependencies = [ + "anyhow", + "crossbeam-channel", + "ide", + "ide-db", + "itertools", + "proc-macro-api", + "project-model", + "tracing", + "tt", + "vfs", + "vfs-notify", +] + +[[package]] name = "lock_api" version = "0.4.10" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -1535,6 +1552,7 @@ dependencies = [ "ide-db", "ide-ssr", "itertools", + "load-cargo", "lsp-server 0.7.1", "lsp-types", "mbe", @@ -1564,7 +1582,6 @@ dependencies = [ "tracing-subscriber", "tracing-tree", "triomphe", - "tt", "vfs", "vfs-notify", "winapi", diff --git a/Cargo.toml b/Cargo.toml index 83d9fd801e..24889fc142 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -61,6 +61,7 @@ ide-diagnostics = { path = "./crates/ide-diagnostics", version = "0.0.0" } ide-ssr = { path = "./crates/ide-ssr", version = "0.0.0" } intern = { path = "./crates/intern", version = "0.0.0" } limit = { path = "./crates/limit", version = "0.0.0" } +load-cargo = { path = "./crates/load-cargo", version = "0.0.0" } mbe = { path = "./crates/mbe", version = "0.0.0" } parser = { path = "./crates/parser", version = "0.0.0" } paths = { path = "./crates/paths", version = "0.0.0" } diff --git a/crates/load-cargo/Cargo.toml b/crates/load-cargo/Cargo.toml new file mode 100644 index 0000000000..4a6946d7f6 --- /dev/null +++ b/crates/load-cargo/Cargo.toml @@ -0,0 +1,23 @@ +[package] +name = "load-cargo" +version = "0.0.0" +rust-version.workspace = true +edition.workspace = true +license.workspace = true +authors.workspace = true + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +anyhow = "1.0.62" +crossbeam-channel = "0.5.5" +itertools = "0.10.5" +tracing = "0.1.35" + +ide.workspace = true +ide-db.workspace =true +proc-macro-api.workspace = true +project-model.workspace = true +tt.workspace = true +vfs.workspace = true +vfs-notify.workspace = true diff --git a/crates/load-cargo/src/lib.rs b/crates/load-cargo/src/lib.rs new file mode 100644 index 0000000000..7a795dd62a --- /dev/null +++ b/crates/load-cargo/src/lib.rs @@ -0,0 +1,441 @@ +//! Loads a Cargo project into a static instance of analysis, without support +//! for incorporating changes. +// Note, don't remove any public api from this. This API is consumed by external tools +// to run rust-analyzer as a library. +use std::{collections::hash_map::Entry, mem, path::Path, sync}; + +use ::tt::token_id as tt; +use crossbeam_channel::{unbounded, Receiver}; +use ide::{AnalysisHost, Change, SourceRoot}; +use ide_db::{ + base_db::{ + CrateGraph, Env, ProcMacro, ProcMacroExpander, ProcMacroExpansionError, ProcMacroKind, + ProcMacroLoadResult, ProcMacros, + }, + FxHashMap, +}; +use itertools::Itertools; +use proc_macro_api::{MacroDylib, ProcMacroServer}; +use project_model::{CargoConfig, PackageRoot, ProjectManifest, ProjectWorkspace}; +use vfs::{file_set::FileSetConfig, loader::Handle, AbsPath, AbsPathBuf, VfsPath}; + +pub struct LoadCargoConfig { + pub load_out_dirs_from_check: bool, + pub with_proc_macro_server: ProcMacroServerChoice, + pub prefill_caches: bool, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum ProcMacroServerChoice { + Sysroot, + Explicit(AbsPathBuf), + None, +} + +pub fn load_workspace_at( + root: &Path, + cargo_config: &CargoConfig, + load_config: &LoadCargoConfig, + progress: &dyn Fn(String), +) -> anyhow::Result<(AnalysisHost, vfs::Vfs, Option<ProcMacroServer>)> { + let root = AbsPathBuf::assert(std::env::current_dir()?.join(root)); + let root = ProjectManifest::discover_single(&root)?; + let mut workspace = ProjectWorkspace::load(root, cargo_config, progress)?; + + if load_config.load_out_dirs_from_check { + let build_scripts = workspace.run_build_scripts(cargo_config, progress)?; + workspace.set_build_scripts(build_scripts) + } + + load_workspace(workspace, &cargo_config.extra_env, load_config) +} + +pub fn load_workspace( + ws: ProjectWorkspace, + extra_env: &FxHashMap<String, String>, + load_config: &LoadCargoConfig, +) -> anyhow::Result<(AnalysisHost, vfs::Vfs, Option<ProcMacroServer>)> { + let (sender, receiver) = unbounded(); + let mut vfs = vfs::Vfs::default(); + let mut loader = { + let loader = + vfs_notify::NotifyHandle::spawn(Box::new(move |msg| sender.send(msg).unwrap())); + Box::new(loader) + }; + + let proc_macro_server = match &load_config.with_proc_macro_server { + ProcMacroServerChoice::Sysroot => ws + .find_sysroot_proc_macro_srv() + .and_then(|it| ProcMacroServer::spawn(it).map_err(Into::into)), + ProcMacroServerChoice::Explicit(path) => { + ProcMacroServer::spawn(path.clone()).map_err(Into::into) + } + ProcMacroServerChoice::None => Err(anyhow::format_err!("proc macro server disabled")), + }; + + let (crate_graph, proc_macros) = ws.to_crate_graph( + &mut |path: &AbsPath| { + let contents = loader.load_sync(path); + let path = vfs::VfsPath::from(path.to_path_buf()); + vfs.set_file_contents(path.clone(), contents); + vfs.file_id(&path) + }, + extra_env, + ); + let proc_macros = { + let proc_macro_server = match &proc_macro_server { + Ok(it) => Ok(it), + Err(e) => Err(e.to_string()), + }; + proc_macros + .into_iter() + .map(|(crate_id, path)| { + ( + crate_id, + path.map_or_else( + |_| Err("proc macro crate is missing dylib".to_owned()), + |(_, path)| { + proc_macro_server.as_ref().map_err(Clone::clone).and_then( + |proc_macro_server| load_proc_macro(proc_macro_server, &path, &[]), + ) + }, + ), + ) + }) + .collect() + }; + + let project_folders = ProjectFolders::new(&[ws], &[]); + loader.set_config(vfs::loader::Config { + load: project_folders.load, + watch: vec![], + version: 0, + }); + + let host = load_crate_graph( + crate_graph, + proc_macros, + project_folders.source_root_config, + &mut vfs, + &receiver, + ); + + if load_config.prefill_caches { + host.analysis().parallel_prime_caches(1, |_| {})?; + } + Ok((host, vfs, proc_macro_server.ok())) +} + +#[derive(Default)] +pub struct ProjectFolders { + pub load: Vec<vfs::loader::Entry>, + pub watch: Vec<usize>, + pub source_root_config: SourceRootConfig, +} + +impl ProjectFolders { + pub fn new(workspaces: &[ProjectWorkspace], global_excludes: &[AbsPathBuf]) -> ProjectFolders { + let mut res = ProjectFolders::default(); + let mut fsc = FileSetConfig::builder(); + let mut local_filesets = vec![]; + + // Dedup source roots + // Depending on the project setup, we can have duplicated source roots, or for example in + // the case of the rustc workspace, we can end up with two source roots that are almost the + // same but not quite, like: + // PackageRoot { is_local: false, include: [AbsPathBuf(".../rust/src/tools/miri/cargo-miri")], exclude: [] } + // PackageRoot { + // is_local: true, + // include: [AbsPathBuf(".../rust/src/tools/miri/cargo-miri"), AbsPathBuf(".../rust/build/x86_64-pc-windows-msvc/stage0-tools/x86_64-pc-windows-msvc/release/build/cargo-miri-85801cd3d2d1dae4/out")], + // exclude: [AbsPathBuf(".../rust/src/tools/miri/cargo-miri/.git"), AbsPathBuf(".../rust/src/tools/miri/cargo-miri/target")] + // } + // + // The first one comes from the explicit rustc workspace which points to the rustc workspace itself + // The second comes from the rustc workspace that we load as the actual project workspace + // These `is_local` differing in this kind of way gives us problems, especially when trying to filter diagnostics as we don't report diagnostics for external libraries. + // So we need to deduplicate these, usually it would be enough to deduplicate by `include`, but as the rustc example shows here that doesn't work, + // so we need to also coalesce the includes if they overlap. + + let mut roots: Vec<_> = workspaces + .iter() + .flat_map(|ws| ws.to_roots()) + .update(|root| root.include.sort()) + .sorted_by(|a, b| a.include.cmp(&b.include)) + .collect(); + + // map that tracks indices of overlapping roots + let mut overlap_map = FxHashMap::<_, Vec<_>>::default(); + let mut done = false; + + while !mem::replace(&mut done, true) { + // maps include paths to indices of the corresponding root + let mut include_to_idx = FxHashMap::default(); + // Find and note down the indices of overlapping roots + for (idx, root) in roots.iter().enumerate().filter(|(_, it)| !it.include.is_empty()) { + for include in &root.include { + match include_to_idx.entry(include) { + Entry::Occupied(e) => { + overlap_map.entry(*e.get()).or_default().push(idx); + } + Entry::Vacant(e) => { + e.insert(idx); + } + } + } + } + for (k, v) in overlap_map.drain() { + done = false; + for v in v { + let r = mem::replace( + &mut roots[v], + PackageRoot { is_local: false, include: vec![], exclude: vec![] }, + ); + roots[k].is_local |= r.is_local; + roots[k].include.extend(r.include); + roots[k].exclude.extend(r.exclude); + } + roots[k].include.sort(); + roots[k].exclude.sort(); + roots[k].include.dedup(); + roots[k].exclude.dedup(); + } + } + + for root in roots.into_iter().filter(|it| !it.include.is_empty()) { + let file_set_roots: Vec<VfsPath> = + root.include.iter().cloned().map(VfsPath::from).collect(); + + let entry = { + let mut dirs = vfs::loader::Directories::default(); + dirs.extensions.push("rs".into()); + dirs.include.extend(root.include); + dirs.exclude.extend(root.exclude); + for excl in global_excludes { + if dirs + .include + .iter() + .any(|incl| incl.starts_with(excl) || excl.starts_with(incl)) + { + dirs.exclude.push(excl.clone()); + } + } + + vfs::loader::Entry::Directories(dirs) + }; + + if root.is_local { + res.watch.push(res.load.len()); + } + res.load.push(entry); + + if root.is_local { + local_filesets.push(fsc.len()); + } + fsc.add_file_set(file_set_roots) + } + + let fsc = fsc.build(); + res.source_root_config = SourceRootConfig { fsc, local_filesets }; + + res + } +} + +#[derive(Default, Debug)] +pub struct SourceRootConfig { + pub fsc: FileSetConfig, + pub local_filesets: Vec<usize>, +} + +impl SourceRootConfig { + pub fn partition(&self, vfs: &vfs::Vfs) -> Vec<SourceRoot> { + self.fsc + .partition(vfs) + .into_iter() + .enumerate() + .map(|(idx, file_set)| { + let is_local = self.local_filesets.contains(&idx); + if is_local { + SourceRoot::new_local(file_set) + } else { + SourceRoot::new_library(file_set) + } + }) + .collect() + } +} + +/// Load the proc-macros for the given lib path, replacing all expanders whose names are in `dummy_replace` +/// with an identity dummy expander. +pub fn load_proc_macro( + server: &ProcMacroServer, + path: &AbsPath, + dummy_replace: &[Box<str>], +) -> ProcMacroLoadResult { + let res: Result<Vec<_>, String> = (|| { + let dylib = MacroDylib::new(path.to_path_buf()); + let vec = server.load_dylib(dylib).map_err(|e| format!("{e}"))?; + if vec.is_empty() { + return Err("proc macro library returned no proc macros".to_string()); + } + Ok(vec + .into_iter() + .map(|expander| expander_to_proc_macro(expander, dummy_replace)) + .collect()) + })(); + match res { + Ok(proc_macros) => { + tracing::info!( + "Loaded proc-macros for {path}: {:?}", + proc_macros.iter().map(|it| it.name.clone()).collect::<Vec<_>>() + ); + Ok(proc_macros) + } + Err(e) => { + tracing::warn!("proc-macro loading for {path} failed: {e}"); + Err(e) + } + } +} + +fn load_crate_graph( + crate_graph: CrateGraph, + proc_macros: ProcMacros, + source_root_config: SourceRootConfig, + vfs: &mut vfs::Vfs, + receiver: &Receiver<vfs::loader::Message>, +) -> AnalysisHost { + let lru_cap = std::env::var("RA_LRU_CAP").ok().and_then(|it| it.parse::<usize>().ok()); + let mut host = AnalysisHost::new(lru_cap); + let mut analysis_change = Change::new(); + + host.raw_database_mut().enable_proc_attr_macros(); + + // wait until Vfs has loaded all roots + for task in receiver { + match task { + vfs::loader::Message::Progress { n_done, n_total, config_version: _ } => { + if n_done == n_total { + break; + } + } + vfs::loader::Message::Loaded { files } => { + for (path, contents) in files { + vfs.set_file_contents(path.into(), contents); + } + } + } + } + let changes = vfs.take_changes(); + for file in changes { + if file.exists() { + let contents = vfs.file_contents(file.file_id); + if let Ok(text) = std::str::from_utf8(contents) { + analysis_change.change_file(file.file_id, Some(text.into())) + } + } + } + let source_roots = source_root_config.partition(vfs); + analysis_change.set_roots(source_roots); + + analysis_change.set_crate_graph(crate_graph); + analysis_change.set_proc_macros(proc_macros); + + host.apply_change(analysis_change); + host +} + +fn expander_to_proc_macro( + expander: proc_macro_api::ProcMacro, + dummy_replace: &[Box<str>], +) -> ProcMacro { + let name = From::from(expander.name()); + let kind = match expander.kind() { + proc_macro_api::ProcMacroKind::CustomDerive => ProcMacroKind::CustomDerive, + proc_macro_api::ProcMacroKind::FuncLike => ProcMacroKind::FuncLike, + proc_macro_api::ProcMacroKind::Attr => ProcMacroKind::Attr, + }; + let expander: sync::Arc<dyn ProcMacroExpander> = + if dummy_replace.iter().any(|replace| &**replace == name) { + match kind { + ProcMacroKind::Attr => sync::Arc::new(IdentityExpander), + _ => sync::Arc::new(EmptyExpander), + } + } else { + sync::Arc::new(Expander(expander)) + }; + ProcMacro { name, kind, expander } +} + +#[derive(Debug)] +struct Expander(proc_macro_api::ProcMacro); + +impl ProcMacroExpander for Expander { + fn expand( + &self, + subtree: &tt::Subtree, + attrs: Option<&tt::Subtree>, + env: &Env, + ) -> Result<tt::Subtree, ProcMacroExpansionError> { + let env = env.iter().map(|(k, v)| (k.to_string(), v.to_string())).collect(); + match self.0.expand(subtree, attrs, env) { + Ok(Ok(subtree)) => Ok(subtree), + Ok(Err(err)) => Err(ProcMacroExpansionError::Panic(err.0)), + Err(err) => Err(ProcMacroExpansionError::System(err.to_string())), + } + } +} + +/// Dummy identity expander, used for attribute proc-macros that are deliberately ignored by the user. +#[derive(Debug)] +struct IdentityExpander; + +impl ProcMacroExpander for IdentityExpander { + fn expand( + &self, + subtree: &tt::Subtree, + _: Option<&tt::Subtree>, + _: &Env, + ) -> Result<tt::Subtree, ProcMacroExpansionError> { + Ok(subtree.clone()) + } +} + +/// Empty expander, used for proc-macros that are deliberately ignored by the user. +#[derive(Debug)] +struct EmptyExpander; + +impl ProcMacroExpander for EmptyExpander { + fn expand( + &self, + _: &tt::Subtree, + _: Option<&tt::Subtree>, + _: &Env, + ) -> Result<tt::Subtree, ProcMacroExpansionError> { + Ok(tt::Subtree::empty()) + } +} + +#[cfg(test)] +mod tests { + use ide_db::base_db::SourceDatabase; + + use super::*; + + #[test] + fn test_loading_rust_analyzer() { + let path = Path::new(env!("CARGO_MANIFEST_DIR")).parent().unwrap().parent().unwrap(); + let cargo_config = CargoConfig::default(); + let load_cargo_config = LoadCargoConfig { + load_out_dirs_from_check: false, + with_proc_macro_server: ProcMacroServerChoice::None, + prefill_caches: false, + }; + let (host, _vfs, _proc_macro) = + load_workspace_at(path, &cargo_config, &load_cargo_config, &|_| {}).unwrap(); + + let n_crates = host.raw_database().crate_graph().iter().count(); + // RA has quite a few crates, but the exact count doesn't matter + assert!(n_crates > 20); + } +} diff --git a/crates/rust-analyzer/Cargo.toml b/crates/rust-analyzer/Cargo.toml index 14104b0955..5bfac7ee45 100644 --- a/crates/rust-analyzer/Cargo.toml +++ b/crates/rust-analyzer/Cargo.toml @@ -62,13 +62,13 @@ ide-db.workspace = true # This should only be used in CLI ide-ssr.workspace = true ide.workspace = true +load-cargo.workspace = true proc-macro-api.workspace = true profile.workspace = true project-model.workspace = true stdx.workspace = true syntax.workspace = true toolchain.workspace = true -tt.workspace = true vfs-notify.workspace = true vfs.workspace = true diff --git a/crates/rust-analyzer/src/cli.rs b/crates/rust-analyzer/src/cli.rs index 893eadf3fa..64646b33ad 100644 --- a/crates/rust-analyzer/src/cli.rs +++ b/crates/rust-analyzer/src/cli.rs @@ -1,7 +1,6 @@ //! Various batch processing tasks, intended primarily for debugging. pub mod flags; -pub mod load_cargo; mod parse; mod symbols; mod highlight; diff --git a/crates/rust-analyzer/src/cli/analysis_stats.rs b/crates/rust-analyzer/src/cli/analysis_stats.rs index 1ceda8d824..7016bd0046 100644 --- a/crates/rust-analyzer/src/cli/analysis_stats.rs +++ b/crates/rust-analyzer/src/cli/analysis_stats.rs @@ -24,6 +24,7 @@ use ide_db::{ LineIndexDatabase, }; use itertools::Itertools; +use load_cargo::{load_workspace, LoadCargoConfig, ProcMacroServerChoice}; use oorandom::Rand32; use profile::{Bytes, StopWatch}; use project_model::{CargoConfig, ProjectManifest, ProjectWorkspace, RustLibSource}; @@ -34,9 +35,7 @@ use vfs::{AbsPathBuf, FileId, Vfs, VfsPath}; use crate::cli::{ flags::{self, OutputFormat}, - full_name_of_item, - load_cargo::{load_workspace, LoadCargoConfig, ProcMacroServerChoice}, - print_memory_usage, + full_name_of_item, print_memory_usage, progress_report::ProgressReport, report_metric, Verbosity, }; diff --git a/crates/rust-analyzer/src/cli/diagnostics.rs b/crates/rust-analyzer/src/cli/diagnostics.rs index 0b46f35074..0db5fb4740 100644 --- a/crates/rust-analyzer/src/cli/diagnostics.rs +++ b/crates/rust-analyzer/src/cli/diagnostics.rs @@ -7,11 +7,9 @@ use rustc_hash::FxHashSet; use hir::{db::HirDatabase, Crate, Module}; use ide::{AssistResolveStrategy, DiagnosticsConfig, Severity}; use ide_db::base_db::SourceDatabaseExt; +use load_cargo::{load_workspace_at, LoadCargoConfig, ProcMacroServerChoice}; -use crate::cli::{ - flags, - load_cargo::{load_workspace_at, LoadCargoConfig, ProcMacroServerChoice}, -}; +use crate::cli::flags; impl flags::Diagnostics { pub fn run(self) -> anyhow::Result<()> { diff --git a/crates/rust-analyzer/src/cli/load_cargo.rs b/crates/rust-analyzer/src/cli/load_cargo.rs deleted file mode 100644 index 1b413029e0..0000000000 --- a/crates/rust-analyzer/src/cli/load_cargo.rs +++ /dev/null @@ -1,204 +0,0 @@ -//! Loads a Cargo project into a static instance of analysis, without support -//! for incorporating changes. -use std::path::Path; - -use crossbeam_channel::{unbounded, Receiver}; -use ide::{AnalysisHost, Change}; -use ide_db::{ - base_db::{CrateGraph, ProcMacros}, - FxHashMap, -}; -use proc_macro_api::ProcMacroServer; -use project_model::{CargoConfig, ProjectManifest, ProjectWorkspace}; -use triomphe::Arc; -use vfs::{loader::Handle, AbsPath, AbsPathBuf}; - -use crate::reload::{load_proc_macro, ProjectFolders, SourceRootConfig}; - -// Note: Since this type is used by external tools that use rust-analyzer as a library -// what otherwise would be `pub(crate)` has to be `pub` here instead. -pub struct LoadCargoConfig { - pub load_out_dirs_from_check: bool, - pub with_proc_macro_server: ProcMacroServerChoice, - pub prefill_caches: bool, -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum ProcMacroServerChoice { - Sysroot, - Explicit(AbsPathBuf), - None, -} - -// Note: Since this function is used by external tools that use rust-analyzer as a library -// what otherwise would be `pub(crate)` has to be `pub` here instead. -pub fn load_workspace_at( - root: &Path, - cargo_config: &CargoConfig, - load_config: &LoadCargoConfig, - progress: &dyn Fn(String), -) -> anyhow::Result<(AnalysisHost, vfs::Vfs, Option<ProcMacroServer>)> { - let root = AbsPathBuf::assert(std::env::current_dir()?.join(root)); - let root = ProjectManifest::discover_single(&root)?; - let mut workspace = ProjectWorkspace::load(root, cargo_config, progress)?; - - if load_config.load_out_dirs_from_check { - let build_scripts = workspace.run_build_scripts(cargo_config, progress)?; - workspace.set_build_scripts(build_scripts) - } - - load_workspace(workspace, &cargo_config.extra_env, load_config) -} - -// Note: Since this function is used by external tools that use rust-analyzer as a library -// what otherwise would be `pub(crate)` has to be `pub` here instead. -// -// The reason both, `load_workspace_at` and `load_workspace` are `pub` is that some of -// these tools need access to `ProjectWorkspace`, too, which `load_workspace_at` hides. -pub fn load_workspace( - ws: ProjectWorkspace, - extra_env: &FxHashMap<String, String>, - load_config: &LoadCargoConfig, -) -> anyhow::Result<(AnalysisHost, vfs::Vfs, Option<ProcMacroServer>)> { - let (sender, receiver) = unbounded(); - let mut vfs = vfs::Vfs::default(); - let mut loader = { - let loader = - vfs_notify::NotifyHandle::spawn(Box::new(move |msg| sender.send(msg).unwrap())); - Box::new(loader) - }; - - let proc_macro_server = match &load_config.with_proc_macro_server { - ProcMacroServerChoice::Sysroot => ws - .find_sysroot_proc_macro_srv() - .and_then(|it| ProcMacroServer::spawn(it).map_err(Into::into)), - ProcMacroServerChoice::Explicit(path) => { - ProcMacroServer::spawn(path.clone()).map_err(Into::into) - } - ProcMacroServerChoice::None => Err(anyhow::format_err!("proc macro server disabled")), - }; - - let (crate_graph, proc_macros) = ws.to_crate_graph( - &mut |path: &AbsPath| { - let contents = loader.load_sync(path); - let path = vfs::VfsPath::from(path.to_path_buf()); - vfs.set_file_contents(path.clone(), contents); - vfs.file_id(&path) - }, - extra_env, - ); - let proc_macros = { - let proc_macro_server = match &proc_macro_server { - Ok(it) => Ok(it), - Err(e) => Err(e.to_string()), - }; - proc_macros - .into_iter() - .map(|(crate_id, path)| { - ( - crate_id, - path.map_or_else( - |_| Err("proc macro crate is missing dylib".to_owned()), - |(_, path)| { - proc_macro_server.as_ref().map_err(Clone::clone).and_then( - |proc_macro_server| load_proc_macro(proc_macro_server, &path, &[]), - ) - }, - ), - ) - }) - .collect() - }; - - let project_folders = ProjectFolders::new(&[ws], &[]); - loader.set_config(vfs::loader::Config { - load: project_folders.load, - watch: vec![], - version: 0, - }); - - tracing::debug!("crate graph: {:?}", crate_graph); - let host = load_crate_graph( - crate_graph, - proc_macros, - project_folders.source_root_config, - &mut vfs, - &receiver, - ); - - if load_config.prefill_caches { - host.analysis().parallel_prime_caches(1, |_| {})?; - } - Ok((host, vfs, proc_macro_server.ok())) -} - -fn load_crate_graph( - crate_graph: CrateGraph, - proc_macros: ProcMacros, - source_root_config: SourceRootConfig, - vfs: &mut vfs::Vfs, - receiver: &Receiver<vfs::loader::Message>, -) -> AnalysisHost { - let lru_cap = std::env::var("RA_LRU_CAP").ok().and_then(|it| it.parse::<usize>().ok()); - let mut host = AnalysisHost::new(lru_cap); - let mut analysis_change = Change::new(); - - host.raw_database_mut().enable_proc_attr_macros(); - - // wait until Vfs has loaded all roots - for task in receiver { - match task { - vfs::loader::Message::Progress { n_done, n_total, config_version: _ } => { - if n_done == n_total { - break; - } - } - vfs::loader::Message::Loaded { files } => { - for (path, contents) in files { - vfs.set_file_contents(path.into(), contents); - } - } - } - } - let changes = vfs.take_changes(); - for file in changes { - if file.exists() { - let contents = vfs.file_contents(file.file_id); - if let Ok(text) = std::str::from_utf8(contents) { - analysis_change.change_file(file.file_id, Some(Arc::from(text))) - } - } - } - let source_roots = source_root_config.partition(vfs); - analysis_change.set_roots(source_roots); - - analysis_change.set_crate_graph(crate_graph); - analysis_change.set_proc_macros(proc_macros); - - host.apply_change(analysis_change); - host -} - -#[cfg(test)] -mod tests { - use super::*; - - use hir::Crate; - - #[test] - fn test_loading_rust_analyzer() { - let path = Path::new(env!("CARGO_MANIFEST_DIR")).parent().unwrap().parent().unwrap(); - let cargo_config = CargoConfig::default(); - let load_cargo_config = LoadCargoConfig { - load_out_dirs_from_check: false, - with_proc_macro_server: ProcMacroServerChoice::None, - prefill_caches: false, - }; - let (host, _vfs, _proc_macro) = - load_workspace_at(path, &cargo_config, &load_cargo_config, &|_| {}).unwrap(); - - let n_crates = Crate::all(host.raw_database()).len(); - // RA has quite a few crates, but the exact count doesn't matter - assert!(n_crates > 20); - } -} diff --git a/crates/rust-analyzer/src/cli/lsif.rs b/crates/rust-analyzer/src/cli/lsif.rs index 6506f4cc4f..bb5016349a 100644 --- a/crates/rust-analyzer/src/cli/lsif.rs +++ b/crates/rust-analyzer/src/cli/lsif.rs @@ -8,22 +8,22 @@ use ide::{ Analysis, FileId, FileRange, MonikerKind, PackageInformation, RootDatabase, StaticIndex, StaticIndexedFile, TokenId, TokenStaticData, }; -use ide_db::LineIndexDatabase; - -use ide_db::base_db::salsa::{self, ParallelDatabase}; -use ide_db::line_index::WideEncoding; +use ide_db::{ + base_db::salsa::{self, ParallelDatabase}, + line_index::WideEncoding, + LineIndexDatabase, +}; +use load_cargo::{load_workspace, LoadCargoConfig, ProcMacroServerChoice}; use lsp_types::{self, lsif}; use project_model::{CargoConfig, ProjectManifest, ProjectWorkspace, RustLibSource}; use vfs::{AbsPathBuf, Vfs}; -use crate::cli::load_cargo::ProcMacroServerChoice; -use crate::cli::{ - flags, - load_cargo::{load_workspace, LoadCargoConfig}, +use crate::{ + cli::flags, + line_index::{LineEndings, LineIndex, PositionEncoding}, + to_proto, + version::version, }; -use crate::line_index::{LineEndings, LineIndex, PositionEncoding}; -use crate::to_proto; -use crate::version::version; /// Need to wrap Snapshot to provide `Clone` impl for `map_with` struct Snap<DB>(DB); diff --git a/crates/rust-analyzer/src/cli/run_tests.rs b/crates/rust-analyzer/src/cli/run_tests.rs index bebbf26b54..b63a266a57 100644 --- a/crates/rust-analyzer/src/cli/run_tests.rs +++ b/crates/rust-analyzer/src/cli/run_tests.rs @@ -7,12 +7,9 @@ use profile::StopWatch; use project_model::{CargoConfig, RustLibSource}; use syntax::TextRange; -use crate::cli::{ - flags, full_name_of_item, - load_cargo::load_workspace_at, - load_cargo::{LoadCargoConfig, ProcMacroServerChoice}, - Result, -}; +use load_cargo::{load_workspace_at, LoadCargoConfig, ProcMacroServerChoice}; + +use crate::cli::{flags, full_name_of_item, Result}; impl flags::RunTests { pub fn run(self) -> Result<()> { diff --git a/crates/rust-analyzer/src/cli/scip.rs b/crates/rust-analyzer/src/cli/scip.rs index 3dd2dc69a1..4579aca302 100644 --- a/crates/rust-analyzer/src/cli/scip.rs +++ b/crates/rust-analyzer/src/cli/scip.rs @@ -6,22 +6,19 @@ use std::{ time::Instant, }; -use crate::{ - cli::load_cargo::ProcMacroServerChoice, - line_index::{LineEndings, LineIndex, PositionEncoding}, -}; use ide::{ LineCol, MonikerDescriptorKind, StaticIndex, StaticIndexedFile, TextRange, TokenId, TokenStaticData, }; use ide_db::LineIndexDatabase; +use load_cargo::{load_workspace, LoadCargoConfig, ProcMacroServerChoice}; use project_model::{CargoConfig, ProjectManifest, ProjectWorkspace, RustLibSource}; use scip::types as scip_types; use std::env; -use crate::cli::{ - flags, - load_cargo::{load_workspace, LoadCargoConfig}, +use crate::{ + cli::flags, + line_index::{LineEndings, LineIndex, PositionEncoding}, }; impl flags::Scip { @@ -275,7 +272,7 @@ mod test { let change_fixture = ChangeFixture::parse(ra_fixture); host.raw_database_mut().apply_change(change_fixture.change); let (file_id, range_or_offset) = - change_fixture.file_position.expect("expected a marker ($0)"); + change_fixture.file_position.expect("expected a marker ()"); let offset = range_or_offset.expect_offset(); (host, FilePosition { file_id, offset }) } diff --git a/crates/rust-analyzer/src/cli/ssr.rs b/crates/rust-analyzer/src/cli/ssr.rs index 7ccb3e216a..f87dcb889a 100644 --- a/crates/rust-analyzer/src/cli/ssr.rs +++ b/crates/rust-analyzer/src/cli/ssr.rs @@ -2,12 +2,10 @@ use anyhow::Context; use ide_ssr::MatchFinder; +use load_cargo::{load_workspace_at, LoadCargoConfig, ProcMacroServerChoice}; use project_model::{CargoConfig, RustLibSource}; -use crate::cli::{ - flags, - load_cargo::{load_workspace_at, LoadCargoConfig, ProcMacroServerChoice}, -}; +use crate::cli::flags; impl flags::Ssr { pub fn run(self) -> anyhow::Result<()> { diff --git a/crates/rust-analyzer/src/global_state.rs b/crates/rust-analyzer/src/global_state.rs index 19a931f65b..ea8a697519 100644 --- a/crates/rust-analyzer/src/global_state.rs +++ b/crates/rust-analyzer/src/global_state.rs @@ -9,6 +9,7 @@ use crossbeam_channel::{unbounded, Receiver, Sender}; use flycheck::FlycheckHandle; use ide::{Analysis, AnalysisHost, Cancellable, Change, FileId}; use ide_db::base_db::{CrateId, FileLoader, ProcMacroPaths, SourceDatabase}; +use load_cargo::SourceRootConfig; use lsp_types::{SemanticTokens, Url}; use nohash_hasher::IntMap; use parking_lot::{Mutex, RwLock}; @@ -27,7 +28,7 @@ use crate::{ main_loop::Task, mem_docs::MemDocs, op_queue::OpQueue, - reload::{self, SourceRootConfig}, + reload, task_pool::TaskPool, to_proto::url_from_abs_path, }; diff --git a/crates/rust-analyzer/src/integrated_benchmarks.rs b/crates/rust-analyzer/src/integrated_benchmarks.rs index bd9f471a46..5a11012b93 100644 --- a/crates/rust-analyzer/src/integrated_benchmarks.rs +++ b/crates/rust-analyzer/src/integrated_benchmarks.rs @@ -20,7 +20,7 @@ use test_utils::project_root; use triomphe::Arc; use vfs::{AbsPathBuf, VfsPath}; -use crate::cli::load_cargo::{load_workspace_at, LoadCargoConfig, ProcMacroServerChoice}; +use load_cargo::{load_workspace_at, LoadCargoConfig, ProcMacroServerChoice}; #[test] fn integrated_highlighting_benchmark() { diff --git a/crates/rust-analyzer/src/reload.rs b/crates/rust-analyzer/src/reload.rs index f9e6152f5b..dd5517e697 100644 --- a/crates/rust-analyzer/src/reload.rs +++ b/crates/rust-analyzer/src/reload.rs @@ -12,26 +12,22 @@ //! correct. Instead, we try to provide a best-effort service. Even if the //! project is currently loading and we don't have a full project model, we //! still want to respond to various requests. -use std::{collections::hash_map::Entry, iter, mem, sync}; +use std::{iter, mem}; use flycheck::{FlycheckConfig, FlycheckHandle}; use hir::db::DefDatabase; use ide::Change; use ide_db::{ - base_db::{ - salsa::Durability, CrateGraph, Env, ProcMacro, ProcMacroExpander, ProcMacroExpansionError, - ProcMacroKind, ProcMacroLoadResult, ProcMacroPaths, ProcMacros, SourceRoot, VfsPath, - }, + base_db::{salsa::Durability, CrateGraph, ProcMacroPaths, ProcMacros}, FxHashMap, }; -use itertools::Itertools; -use proc_macro_api::{MacroDylib, ProcMacroServer}; -use project_model::{PackageRoot, ProjectWorkspace, WorkspaceBuildScripts}; +use load_cargo::{load_proc_macro, ProjectFolders}; +use proc_macro_api::ProcMacroServer; +use project_model::{ProjectWorkspace, WorkspaceBuildScripts}; use rustc_hash::FxHashSet; use stdx::{format_to, thread::ThreadIntent}; -use syntax::SmolStr; use triomphe::Arc; -use vfs::{file_set::FileSetConfig, AbsPath, AbsPathBuf, ChangeKind}; +use vfs::{AbsPath, ChangeKind}; use crate::{ config::{Config, FilesWatcher, LinkedProject}, @@ -41,8 +37,6 @@ use crate::{ op_queue::Cause, }; -use ::tt::token_id as tt; - #[derive(Debug)] pub(crate) enum ProjectWorkspaceProgress { Begin, @@ -619,253 +613,6 @@ impl GlobalState { } } -#[derive(Default)] -pub(crate) struct ProjectFolders { - pub(crate) load: Vec<vfs::loader::Entry>, - pub(crate) watch: Vec<usize>, - pub(crate) source_root_config: SourceRootConfig, -} - -impl ProjectFolders { - pub(crate) fn new( - workspaces: &[ProjectWorkspace], - global_excludes: &[AbsPathBuf], - ) -> ProjectFolders { - let mut res = ProjectFolders::default(); - let mut fsc = FileSetConfig::builder(); - let mut local_filesets = vec![]; - - // Dedup source roots - // Depending on the project setup, we can have duplicated source roots, or for example in - // the case of the rustc workspace, we can end up with two source roots that are almost the - // same but not quite, like: - // PackageRoot { is_local: false, include: [AbsPathBuf(".../rust/src/tools/miri/cargo-miri")], exclude: [] } - // PackageRoot { - // is_local: true, - // include: [AbsPathBuf(".../rust/src/tools/miri/cargo-miri"), AbsPathBuf(".../rust/build/x86_64-pc-windows-msvc/stage0-tools/x86_64-pc-windows-msvc/release/build/cargo-miri-85801cd3d2d1dae4/out")], - // exclude: [AbsPathBuf(".../rust/src/tools/miri/cargo-miri/.git"), AbsPathBuf(".../rust/src/tools/miri/cargo-miri/target")] - // } - // - // The first one comes from the explicit rustc workspace which points to the rustc workspace itself - // The second comes from the rustc workspace that we load as the actual project workspace - // These `is_local` differing in this kind of way gives us problems, especially when trying to filter diagnostics as we don't report diagnostics for external libraries. - // So we need to deduplicate these, usually it would be enough to deduplicate by `include`, but as the rustc example shows here that doesn't work, - // so we need to also coalesce the includes if they overlap. - - let mut roots: Vec<_> = workspaces - .iter() - .flat_map(|ws| ws.to_roots()) - .update(|root| root.include.sort()) - .sorted_by(|a, b| a.include.cmp(&b.include)) - .collect(); - - // map that tracks indices of overlapping roots - let mut overlap_map = FxHashMap::<_, Vec<_>>::default(); - let mut done = false; - - while !mem::replace(&mut done, true) { - // maps include paths to indices of the corresponding root - let mut include_to_idx = FxHashMap::default(); - // Find and note down the indices of overlapping roots - for (idx, root) in roots.iter().enumerate().filter(|(_, it)| !it.include.is_empty()) { - for include in &root.include { - match include_to_idx.entry(include) { - Entry::Occupied(e) => { - overlap_map.entry(*e.get()).or_default().push(idx); - } - Entry::Vacant(e) => { - e.insert(idx); - } - } - } - } - for (k, v) in overlap_map.drain() { - done = false; - for v in v { - let r = mem::replace( - &mut roots[v], - PackageRoot { is_local: false, include: vec![], exclude: vec![] }, - ); - roots[k].is_local |= r.is_local; - roots[k].include.extend(r.include); - roots[k].exclude.extend(r.exclude); - } - roots[k].include.sort(); - roots[k].exclude.sort(); - roots[k].include.dedup(); - roots[k].exclude.dedup(); - } - } - - for root in roots.into_iter().filter(|it| !it.include.is_empty()) { - let file_set_roots: Vec<VfsPath> = - root.include.iter().cloned().map(VfsPath::from).collect(); - - let entry = { - let mut dirs = vfs::loader::Directories::default(); - dirs.extensions.push("rs".into()); - dirs.include.extend(root.include); - dirs.exclude.extend(root.exclude); - for excl in global_excludes { - if dirs - .include - .iter() - .any(|incl| incl.starts_with(excl) || excl.starts_with(incl)) - { - dirs.exclude.push(excl.clone()); - } - } - - vfs::loader::Entry::Directories(dirs) - }; - - if root.is_local { - res.watch.push(res.load.len()); - } - res.load.push(entry); - - if root.is_local { - local_filesets.push(fsc.len()); - } - fsc.add_file_set(file_set_roots) - } - - let fsc = fsc.build(); - res.source_root_config = SourceRootConfig { fsc, local_filesets }; - - res - } -} - -#[derive(Default, Debug)] -pub(crate) struct SourceRootConfig { - pub(crate) fsc: FileSetConfig, - pub(crate) local_filesets: Vec<usize>, -} - -impl SourceRootConfig { - pub(crate) fn partition(&self, vfs: &vfs::Vfs) -> Vec<SourceRoot> { - let _p = profile::span("SourceRootConfig::partition"); - self.fsc - .partition(vfs) - .into_iter() - .enumerate() - .map(|(idx, file_set)| { - let is_local = self.local_filesets.contains(&idx); - if is_local { - SourceRoot::new_local(file_set) - } else { - SourceRoot::new_library(file_set) - } - }) - .collect() - } -} - -/// Load the proc-macros for the given lib path, replacing all expanders whose names are in `dummy_replace` -/// with an identity dummy expander. -pub(crate) fn load_proc_macro( - server: &ProcMacroServer, - path: &AbsPath, - dummy_replace: &[Box<str>], -) -> ProcMacroLoadResult { - let res: Result<Vec<_>, String> = (|| { - let dylib = MacroDylib::new(path.to_path_buf()); - let vec = server.load_dylib(dylib).map_err(|e| format!("{e}"))?; - if vec.is_empty() { - return Err("proc macro library returned no proc macros".to_string()); - } - Ok(vec - .into_iter() - .map(|expander| expander_to_proc_macro(expander, dummy_replace)) - .collect()) - })(); - return match res { - Ok(proc_macros) => { - tracing::info!( - "Loaded proc-macros for {path}: {:?}", - proc_macros.iter().map(|it| it.name.clone()).collect::<Vec<_>>() - ); - Ok(proc_macros) - } - Err(e) => { - tracing::warn!("proc-macro loading for {path} failed: {e}"); - Err(e) - } - }; - - fn expander_to_proc_macro( - expander: proc_macro_api::ProcMacro, - dummy_replace: &[Box<str>], - ) -> ProcMacro { - let name = SmolStr::from(expander.name()); - let kind = match expander.kind() { - proc_macro_api::ProcMacroKind::CustomDerive => ProcMacroKind::CustomDerive, - proc_macro_api::ProcMacroKind::FuncLike => ProcMacroKind::FuncLike, - proc_macro_api::ProcMacroKind::Attr => ProcMacroKind::Attr, - }; - let expander: sync::Arc<dyn ProcMacroExpander> = - if dummy_replace.iter().any(|replace| &**replace == name) { - match kind { - ProcMacroKind::Attr => sync::Arc::new(IdentityExpander), - _ => sync::Arc::new(EmptyExpander), - } - } else { - sync::Arc::new(Expander(expander)) - }; - ProcMacro { name, kind, expander } - } - - #[derive(Debug)] - struct Expander(proc_macro_api::ProcMacro); - - impl ProcMacroExpander for Expander { - fn expand( - &self, - subtree: &tt::Subtree, - attrs: Option<&tt::Subtree>, - env: &Env, - ) -> Result<tt::Subtree, ProcMacroExpansionError> { - let env = env.iter().map(|(k, v)| (k.to_string(), v.to_string())).collect(); - match self.0.expand(subtree, attrs, env) { - Ok(Ok(subtree)) => Ok(subtree), - Ok(Err(err)) => Err(ProcMacroExpansionError::Panic(err.0)), - Err(err) => Err(ProcMacroExpansionError::System(err.to_string())), - } - } - } - - /// Dummy identity expander, used for attribute proc-macros that are deliberately ignored by the user. - #[derive(Debug)] - struct IdentityExpander; - - impl ProcMacroExpander for IdentityExpander { - fn expand( - &self, - subtree: &tt::Subtree, - _: Option<&tt::Subtree>, - _: &Env, - ) -> Result<tt::Subtree, ProcMacroExpansionError> { - Ok(subtree.clone()) - } - } - - /// Empty expander, used for proc-macros that are deliberately ignored by the user. - #[derive(Debug)] - struct EmptyExpander; - - impl ProcMacroExpander for EmptyExpander { - fn expand( - &self, - _: &tt::Subtree, - _: Option<&tt::Subtree>, - _: &Env, - ) -> Result<tt::Subtree, ProcMacroExpansionError> { - Ok(tt::Subtree::empty()) - } - } -} - pub(crate) fn should_refresh_for_change(path: &AbsPath, change_kind: ChangeKind) -> bool { const IMPLICIT_TARGET_FILES: &[&str] = &["build.rs", "src/main.rs", "src/lib.rs"]; const IMPLICIT_TARGET_DIRS: &[&str] = &["src/bin", "examples", "tests", "benches"]; |