Unnamed repository; edit this file 'description' to name the repository.
Diffstat (limited to 'helix-loader/src/grammar.rs')
| -rw-r--r-- | helix-loader/src/grammar.rs | 249 |
1 files changed, 77 insertions, 172 deletions
diff --git a/helix-loader/src/grammar.rs b/helix-loader/src/grammar.rs index 11ddc0e4..2aa92475 100644 --- a/helix-loader/src/grammar.rs +++ b/helix-loader/src/grammar.rs @@ -1,4 +1,4 @@ -use anyhow::{anyhow, bail, Context, Result}; +use anyhow::{anyhow, Context, Result}; use serde::{Deserialize, Serialize}; use std::fs; use std::time::SystemTime; @@ -8,8 +8,7 @@ use std::{ process::Command, sync::mpsc::channel, }; -use tempfile::TempPath; -use tree_house::tree_sitter::Grammar; +use tree_sitter::Language; #[cfg(unix)] const DYLIB_EXTENSION: &str = "so"; @@ -61,31 +60,30 @@ const BUILD_TARGET: &str = env!("BUILD_TARGET"); const REMOTE_NAME: &str = "origin"; #[cfg(target_arch = "wasm32")] -pub fn get_language(name: &str) -> Result<Option<Grammar>> { +pub fn get_language(name: &str) -> Result<Language> { unimplemented!() } #[cfg(not(target_arch = "wasm32"))] -pub fn get_language(name: &str) -> Result<Option<Grammar>> { - let mut rel_library_path = PathBuf::new().join("grammars").join(name); - rel_library_path.set_extension(DYLIB_EXTENSION); - let library_path = crate::runtime_file(&rel_library_path); - if !library_path.exists() { - return Ok(None); - } - - let grammar = unsafe { Grammar::new(name, &library_path) }?; - Ok(Some(grammar)) -} +pub fn get_language(name: &str) -> Result<Language> { + use libloading::{Library, Symbol}; + let mut library_path = crate::runtime_dir().join("grammars").join(name); + library_path.set_extension(DYLIB_EXTENSION); -fn ensure_git_is_available() -> Result<()> { - helix_stdx::env::which("git")?; - Ok(()) + let library = unsafe { Library::new(&library_path) } + .with_context(|| format!("Error opening dynamic library {:?}", library_path))?; + let language_fn_name = format!("tree_sitter_{}", name.replace('-', "_")); + let language = unsafe { + let language_fn: Symbol<unsafe extern "C" fn() -> Language> = library + .get(language_fn_name.as_bytes()) + .with_context(|| format!("Failed to load symbol {}", language_fn_name))?; + language_fn() + }; + std::mem::forget(library); + Ok(language) } pub fn fetch_grammars() -> Result<()> { - ensure_git_is_available()?; - // We do not need to fetch local grammars. let mut grammars = get_grammar_configs()?; grammars.retain(|grammar| !matches!(grammar.source, GrammarSource::Local { .. })); @@ -98,12 +96,15 @@ pub fn fetch_grammars() -> Result<()> { let mut git_up_to_date = 0; let mut non_git = Vec::new(); - for (grammar_id, res) in results { + for res in results { match res { Ok(FetchStatus::GitUpToDate) => git_up_to_date += 1, - Ok(FetchStatus::GitUpdated { revision }) => git_updated.push((grammar_id, revision)), - Ok(FetchStatus::NonGit) => non_git.push(grammar_id), - Err(e) => errors.push((grammar_id, e)), + Ok(FetchStatus::GitUpdated { + grammar_id, + revision, + }) => git_updated.push((grammar_id, revision)), + Ok(FetchStatus::NonGit { grammar_id }) => non_git.push(grammar_id), + Err(e) => errors.push(e), } } @@ -135,18 +136,16 @@ pub fn fetch_grammars() -> Result<()> { if !errors.is_empty() { let len = errors.len(); - for (i, (grammar, error)) in errors.into_iter().enumerate() { - println!("Failure {}/{len}: {grammar} {error}", i + 1); + println!("{} grammars failed to fetch", len); + for (i, error) in errors.into_iter().enumerate() { + println!("\tFailure {}/{}: {}", i + 1, len, error); } - bail!("{len} grammars failed to fetch"); } Ok(()) } pub fn build_grammars(target: Option<String>) -> Result<()> { - ensure_git_is_available()?; - let grammars = get_grammar_configs()?; println!("Building {} grammars", grammars.len()); let results = run_parallel(grammars, move |grammar| { @@ -157,11 +156,11 @@ pub fn build_grammars(target: Option<String>) -> Result<()> { let mut already_built = 0; let mut built = Vec::new(); - for (grammar_id, res) in results { + for res in results { match res { Ok(BuildStatus::AlreadyBuilt) => already_built += 1, - Ok(BuildStatus::Built) => built.push(grammar_id), - Err(e) => errors.push((grammar_id, e)), + Ok(BuildStatus::Built { grammar_id }) => built.push(grammar_id), + Err(e) => errors.push(e), } } @@ -178,10 +177,10 @@ pub fn build_grammars(target: Option<String>) -> Result<()> { if !errors.is_empty() { let len = errors.len(); - for (i, (grammar_id, error)) in errors.into_iter().enumerate() { - println!("Failure {}/{len}: {grammar_id} {error}", i + 1); + println!("{} grammars failed to build", len); + for (i, error) in errors.into_iter().enumerate() { + println!("\tFailure {}/{}: {}", i, len, error); } - bail!("{len} grammars failed to build"); } Ok(()) @@ -213,28 +212,7 @@ fn get_grammar_configs() -> Result<Vec<GrammarConfiguration>> { Ok(grammars) } -pub fn get_grammar_names() -> Result<Option<HashSet<String>>> { - let config: Configuration = crate::config::user_lang_config() - .context("Could not parse languages.toml")? - .try_into()?; - - let grammars = match config.grammar_selection { - Some(GrammarSelection::Only { only: selections }) => Some(selections), - Some(GrammarSelection::Except { except: rejections }) => Some( - config - .grammar - .into_iter() - .map(|grammar| grammar.grammar_id) - .filter(|id| !rejections.contains(id)) - .collect(), - ), - None => None, - }; - - Ok(grammars) -} - -fn run_parallel<F, Res>(grammars: Vec<GrammarConfiguration>, job: F) -> Vec<(String, Result<Res>)> +fn run_parallel<F, Res>(grammars: Vec<GrammarConfiguration>, job: F) -> Vec<Result<Res>> where F: Fn(GrammarConfiguration) -> Result<Res> + Send + 'static + Clone, Res: Send + 'static, @@ -249,7 +227,7 @@ where pool.execute(move || { // Ignore any SendErrors, if any job in another thread has encountered an // error the Receiver will be closed causing this send to fail. - let _ = tx.send((grammar.grammar_id.clone(), job(grammar))); + let _ = tx.send(job(grammar)); }); } @@ -260,8 +238,13 @@ where enum FetchStatus { GitUpToDate, - GitUpdated { revision: String }, - NonGit, + GitUpdated { + grammar_id: String, + revision: String, + }, + NonGit { + grammar_id: String, + }, } fn fetch_grammar(grammar: GrammarConfiguration) -> Result<FetchStatus> { @@ -269,9 +252,7 @@ fn fetch_grammar(grammar: GrammarConfiguration) -> Result<FetchStatus> { remote, revision, .. } = grammar.source { - let grammar_dir = crate::runtime_dirs() - .first() - .expect("No runtime directories provided") // guaranteed by post-condition + let grammar_dir = crate::runtime_dir() .join("grammars") .join("sources") .join(&grammar.grammar_id); @@ -287,12 +268,12 @@ fn fetch_grammar(grammar: GrammarConfiguration) -> Result<FetchStatus> { } // ensure the remote matches the configured remote - if get_remote_url(&grammar_dir).as_ref() != Some(&remote) { + if get_remote_url(&grammar_dir).map_or(true, |s| s != remote) { set_remote(&grammar_dir, &remote)?; } // ensure the revision matches the configured revision - if get_revision(&grammar_dir).as_ref() != Some(&revision) { + if get_revision(&grammar_dir).map_or(true, |s| s != revision) { // Fetch the exact revision from the remote. // Supported by server-side git since v2.5.0 (July 2015), // enabled by default on major git hosts. @@ -302,12 +283,17 @@ fn fetch_grammar(grammar: GrammarConfiguration) -> Result<FetchStatus> { )?; git(&grammar_dir, ["checkout", &revision])?; - Ok(FetchStatus::GitUpdated { revision }) + Ok(FetchStatus::GitUpdated { + grammar_id: grammar.grammar_id, + revision, + }) } else { Ok(FetchStatus::GitUpToDate) } } else { - Ok(FetchStatus::NonGit) + Ok(FetchStatus::NonGit { + grammar_id: grammar.grammar_id, + }) } } @@ -357,16 +343,14 @@ where enum BuildStatus { AlreadyBuilt, - Built, + Built { grammar_id: String }, } fn build_grammar(grammar: GrammarConfiguration, target: Option<&str>) -> Result<BuildStatus> { let grammar_dir = if let GrammarSource::Local { path } = &grammar.source { PathBuf::from(&path) } else { - crate::runtime_dirs() - .first() - .expect("No runtime directories provided") // guaranteed by post-condition + crate::runtime_dir() .join("grammars") .join("sources") .join(&grammar.grammar_id) @@ -417,26 +401,11 @@ fn build_tree_sitter_library( None } }; - let parser_lib_path = crate::runtime_dirs() - .first() - .expect("No runtime directories provided") // guaranteed by post-condition - .join("grammars"); + let parser_lib_path = crate::runtime_dir().join("grammars"); let mut library_path = parser_lib_path.join(&grammar.grammar_id); library_path.set_extension(DYLIB_EXTENSION); - // if we are running inside a buildscript emit cargo metadata - // to detect if we are running from a buildscript check some env variables - // that cargo only sets for build scripts - if std::env::var("OUT_DIR").is_ok() && std::env::var("CARGO").is_ok() { - if let Some(scanner_path) = scanner_path.as_ref().and_then(|path| path.to_str()) { - println!("cargo:rerun-if-changed={scanner_path}"); - } - if let Some(parser_path) = parser_path.to_str() { - println!("cargo:rerun-if-changed={parser_path}"); - } - } - - let recompile = needs_recompile(&library_path, &parser_path, scanner_path.as_ref()) + let recompile = needs_recompile(&library_path, &parser_path, &scanner_path) .context("Failed to compare source and binary timestamps")?; if !recompile { @@ -456,51 +425,16 @@ fn build_tree_sitter_library( for (key, value) in compiler.env() { command.env(key, value); } - command.args(compiler.args()); - // used to delay dropping the temporary object file until after the compilation is complete - let _path_guard; - if compiler.is_like_msvc() { + if cfg!(all(windows, target_env = "msvc")) { command .args(["/nologo", "/LD", "/I"]) .arg(header_path) - .arg("/utf-8") - .arg("/std:c11"); + .arg("/Od") + .arg("/utf-8"); if let Some(scanner_path) = scanner_path.as_ref() { - if scanner_path.extension() == Some("c".as_ref()) { - command.arg(scanner_path); - } else { - let mut cpp_command = Command::new(compiler.path()); - cpp_command.current_dir(src_path); - for (key, value) in compiler.env() { - cpp_command.env(key, value); - } - cpp_command.args(compiler.args()); - let object_file = - library_path.with_file_name(format!("{}_scanner.obj", &grammar.grammar_id)); - cpp_command - .args(["/nologo", "/LD", "/I"]) - .arg(header_path) - .arg("/utf-8") - .arg("/std:c++14") - .arg(format!("/Fo{}", object_file.display())) - .arg("/c") - .arg(scanner_path); - let output = cpp_command - .output() - .context("Failed to execute C++ compiler")?; - - if !output.status.success() { - return Err(anyhow!( - "Parser compilation failed.\nStdout: {}\nStderr: {}", - String::from_utf8_lossy(&output.stdout), - String::from_utf8_lossy(&output.stderr) - )); - } - command.arg(&object_file); - _path_guard = TempPath::from_path(object_file); - } + command.arg(scanner_path); } command @@ -508,58 +442,24 @@ fn build_tree_sitter_library( .arg("/link") .arg(format!("/out:{}", library_path.to_str().unwrap())); } else { - #[cfg(not(windows))] - command.arg("-fPIC"); - command .arg("-shared") + .arg("-fPIC") .arg("-fno-exceptions") + .arg("-g") .arg("-I") .arg(header_path) .arg("-o") - .arg(&library_path); - + .arg(&library_path) + .arg("-O3"); if let Some(scanner_path) = scanner_path.as_ref() { if scanner_path.extension() == Some("c".as_ref()) { - command.arg("-xc").arg("-std=c11").arg(scanner_path); + command.arg("-xc").arg("-std=c99").arg(scanner_path); } else { - let mut cpp_command = Command::new(compiler.path()); - cpp_command.current_dir(src_path); - for (key, value) in compiler.env() { - cpp_command.env(key, value); - } - cpp_command.args(compiler.args()); - let object_file = - library_path.with_file_name(format!("{}_scanner.o", &grammar.grammar_id)); - - #[cfg(not(windows))] - cpp_command.arg("-fPIC"); - - cpp_command - .arg("-fno-exceptions") - .arg("-I") - .arg(header_path) - .arg("-o") - .arg(&object_file) - .arg("-std=c++14") - .arg("-c") - .arg(scanner_path); - let output = cpp_command - .output() - .context("Failed to execute C++ compiler")?; - if !output.status.success() { - return Err(anyhow!( - "Parser compilation failed.\nStdout: {}\nStderr: {}", - String::from_utf8_lossy(&output.stdout), - String::from_utf8_lossy(&output.stderr) - )); - } - - command.arg(&object_file); - _path_guard = TempPath::from_path(object_file); + command.arg(scanner_path); } } - command.arg("-xc").arg("-std=c11").arg(parser_path); + command.arg("-xc").arg(parser_path); if cfg!(all( unix, not(any(target_os = "macos", target_os = "illumos")) @@ -579,13 +479,15 @@ fn build_tree_sitter_library( )); } - Ok(BuildStatus::Built) + Ok(BuildStatus::Built { + grammar_id: grammar.grammar_id, + }) } fn needs_recompile( lib_path: &Path, parser_c_path: &Path, - scanner_path: Option<&PathBuf>, + scanner_path: &Option<PathBuf>, ) -> Result<bool> { if !lib_path.exists() { return Ok(true); @@ -609,6 +511,9 @@ fn mtime(path: &Path) -> Result<SystemTime> { /// Gives the contents of a file from a language's `runtime/queries/<lang>` /// directory pub fn load_runtime_file(language: &str, filename: &str) -> Result<String, std::io::Error> { - let path = crate::runtime_file(PathBuf::new().join("queries").join(language).join(filename)); - std::fs::read_to_string(path) + let path = crate::RUNTIME_DIR + .join("queries") + .join(language) + .join(filename); + std::fs::read_to_string(&path) } |