Unnamed repository; edit this file 'description' to name the repository.
Compress file text using lz4 in salsa
| -rw-r--r-- | Cargo.lock | 15 | ||||
| -rw-r--r-- | crates/base-db/Cargo.toml | 2 | ||||
| -rw-r--r-- | crates/base-db/src/change.rs | 2 | ||||
| -rw-r--r-- | crates/base-db/src/lib.rs | 43 | ||||
| -rw-r--r-- | crates/hir-def/src/nameres/tests/incremental.rs | 2 | ||||
| -rw-r--r-- | crates/hir-ty/src/tests.rs | 2 | ||||
| -rw-r--r-- | crates/hir-ty/src/tests/incremental.rs | 2 | ||||
| -rw-r--r-- | crates/ide-db/src/apply_change.rs | 1 | ||||
| -rw-r--r-- | crates/ide-db/src/lib.rs | 5 |
9 files changed, 65 insertions, 9 deletions
diff --git a/Cargo.lock b/Cargo.lock index 903141eee9..e2e0550d7a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -71,6 +71,7 @@ version = "0.0.0" dependencies = [ "cfg", "la-arena 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "lz4_flex", "rustc-hash", "salsa", "semver", @@ -134,9 +135,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.0.89" +version = "1.0.90" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a0ba8f7aaa012f30d5b2861462f6708eccd49c3c39863fe083a308035f63d723" +checksum = "8cd6604a82acf3039f1144f54b8eb34e91ffba622051189e71b781822d5ee1f5" [[package]] name = "cfg" @@ -874,9 +875,9 @@ checksum = "9c198f91728a82281a64e1f4f9eeb25d82cb32a5de251c6bd1b5154d63a8e7bd" [[package]] name = "libloading" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2caa5afb8bf9f3a2652760ce7d4f62d21c4d5a423e68466fca30df82f2330164" +checksum = "0c2a198fb6b0eada2a8df47933734e6d35d350665a33a3593d7164fa52c75c19" dependencies = [ "cfg-if", "windows-targets 0.52.4", @@ -993,6 +994,12 @@ dependencies = [ ] [[package]] +name = "lz4_flex" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "912b45c753ff5f7f5208307e8ace7d2a2e30d024e26d3509f3dce546c044ce15" + +[[package]] name = "mbe" version = "0.0.0" dependencies = [ diff --git a/crates/base-db/Cargo.toml b/crates/base-db/Cargo.toml index 118abf5d6e..4ab99fc33c 100644 --- a/crates/base-db/Cargo.toml +++ b/crates/base-db/Cargo.toml @@ -12,6 +12,8 @@ rust-version.workspace = true doctest = false [dependencies] +lz4_flex = { version = "0.11", default-features = false } + la-arena.workspace = true salsa.workspace = true rustc-hash.workspace = true diff --git a/crates/base-db/src/change.rs b/crates/base-db/src/change.rs index 003ffb24d9..d709fde331 100644 --- a/crates/base-db/src/change.rs +++ b/crates/base-db/src/change.rs @@ -7,7 +7,7 @@ use salsa::Durability; use triomphe::Arc; use vfs::FileId; -use crate::{CrateGraph, SourceDatabaseExt, SourceRoot, SourceRootId}; +use crate::{CrateGraph, SourceDatabaseExt, SourceDatabaseExt2, SourceRoot, SourceRootId}; /// Encapsulate a bunch of raw `.set` calls on the database. #[derive(Default)] diff --git a/crates/base-db/src/lib.rs b/crates/base-db/src/lib.rs index 758d2a45c8..2d3609888b 100644 --- a/crates/base-db/src/lib.rs +++ b/crates/base-db/src/lib.rs @@ -7,6 +7,7 @@ mod input; use std::panic; +use salsa::Durability; use syntax::{ast, Parse, SourceFile}; use triomphe::Arc; @@ -42,6 +43,7 @@ pub trait Upcast<T: ?Sized> { fn upcast(&self) -> &T; } +pub const DEFAULT_FILE_TEXT_LRU_CAP: usize = 16; pub const DEFAULT_PARSE_LRU_CAP: usize = 128; pub const DEFAULT_BORROWCK_LRU_CAP: usize = 1024; @@ -89,7 +91,10 @@ fn parse(db: &dyn SourceDatabase, file_id: FileId) -> Parse<ast::SourceFile> { #[salsa::query_group(SourceDatabaseExtStorage)] pub trait SourceDatabaseExt: SourceDatabase { #[salsa::input] + fn compressed_file_text(&self, file_id: FileId) -> Arc<[u8]>; + fn file_text(&self, file_id: FileId) -> Arc<str>; + /// Path to a file, relative to the root of its source root. /// Source root of the file. #[salsa::input] @@ -101,6 +106,44 @@ pub trait SourceDatabaseExt: SourceDatabase { fn source_root_crates(&self, id: SourceRootId) -> Arc<[CrateId]>; } +fn file_text(db: &dyn SourceDatabaseExt, file_id: FileId) -> Arc<str> { + let bytes = db.compressed_file_text(file_id); + let bytes = + lz4_flex::decompress_size_prepended(&bytes).expect("lz4 decompression should not fail"); + let text = std::str::from_utf8(&bytes).expect("file contents should be valid UTF-8"); + Arc::from(text) +} + +pub trait SourceDatabaseExt2 { + fn set_file_text(&mut self, file_id: FileId, text: Arc<str>) { + self.set_file_text_with_durability(file_id, text, Durability::LOW); + } + + fn set_file_text_with_durability( + &mut self, + file_id: FileId, + text: Arc<str>, + durability: Durability, + ); +} + +impl<Db: ?Sized + SourceDatabaseExt> SourceDatabaseExt2 for Db { + fn set_file_text_with_durability( + &mut self, + file_id: FileId, + text: Arc<str>, + durability: Durability, + ) { + let bytes = text.as_bytes(); + let compressed = lz4_flex::compress_prepend_size(&bytes); + self.set_compressed_file_text_with_durability( + file_id, + Arc::from(compressed.as_slice()), + durability, + ) + } +} + fn source_root_crates(db: &dyn SourceDatabaseExt, id: SourceRootId) -> Arc<[CrateId]> { let graph = db.crate_graph(); let mut crates = graph diff --git a/crates/hir-def/src/nameres/tests/incremental.rs b/crates/hir-def/src/nameres/tests/incremental.rs index 6efced0271..189c7b9261 100644 --- a/crates/hir-def/src/nameres/tests/incremental.rs +++ b/crates/hir-def/src/nameres/tests/incremental.rs @@ -1,4 +1,4 @@ -use base_db::{SourceDatabase, SourceDatabaseExt}; +use base_db::{SourceDatabase, SourceDatabaseExt2 as _}; use test_fixture::WithFixture; use triomphe::Arc; diff --git a/crates/hir-ty/src/tests.rs b/crates/hir-ty/src/tests.rs index 5e159236f4..349da8feb2 100644 --- a/crates/hir-ty/src/tests.rs +++ b/crates/hir-ty/src/tests.rs @@ -12,7 +12,7 @@ mod traits; use std::env; -use base_db::{FileRange, SourceDatabaseExt}; +use base_db::{FileRange, SourceDatabaseExt2 as _}; use expect_test::Expect; use hir_def::{ body::{Body, BodySourceMap, SyntheticSyntax}, diff --git a/crates/hir-ty/src/tests/incremental.rs b/crates/hir-ty/src/tests/incremental.rs index 82d934009f..a3128336e7 100644 --- a/crates/hir-ty/src/tests/incremental.rs +++ b/crates/hir-ty/src/tests/incremental.rs @@ -1,4 +1,4 @@ -use base_db::SourceDatabaseExt; +use base_db::SourceDatabaseExt2 as _; use test_fixture::WithFixture; use triomphe::Arc; diff --git a/crates/ide-db/src/apply_change.rs b/crates/ide-db/src/apply_change.rs index 017635d88e..ec05f6d13d 100644 --- a/crates/ide-db/src/apply_change.rs +++ b/crates/ide-db/src/apply_change.rs @@ -205,6 +205,7 @@ impl RootDatabase { // SourceDatabaseExt base_db::FileTextQuery + base_db::CompressedFileTextQuery base_db::FileSourceRootQuery base_db::SourceRootQuery base_db::SourceRootCratesQuery diff --git a/crates/ide-db/src/lib.rs b/crates/ide-db/src/lib.rs index be08b37bac..79076e68cb 100644 --- a/crates/ide-db/src/lib.rs +++ b/crates/ide-db/src/lib.rs @@ -51,6 +51,7 @@ use std::{fmt, mem::ManuallyDrop}; use base_db::{ salsa::{self, Durability}, AnchoredPath, CrateId, FileId, FileLoader, FileLoaderDelegate, SourceDatabase, Upcast, + DEFAULT_FILE_TEXT_LRU_CAP, }; use hir::db::{DefDatabase, ExpandDatabase, HirDatabase}; use triomphe::Arc; @@ -157,6 +158,7 @@ impl RootDatabase { pub fn update_base_query_lru_capacities(&mut self, lru_capacity: Option<usize>) { let lru_capacity = lru_capacity.unwrap_or(base_db::DEFAULT_PARSE_LRU_CAP); + base_db::FileTextQuery.in_db_mut(self).set_lru_capacity(DEFAULT_FILE_TEXT_LRU_CAP); base_db::ParseQuery.in_db_mut(self).set_lru_capacity(lru_capacity); // macro expansions are usually rather small, so we can afford to keep more of them alive hir::db::ParseMacroExpansionQuery.in_db_mut(self).set_lru_capacity(4 * lru_capacity); @@ -166,6 +168,7 @@ impl RootDatabase { pub fn update_lru_capacities(&mut self, lru_capacities: &FxHashMap<Box<str>, usize>) { use hir::db as hir_db; + base_db::FileTextQuery.in_db_mut(self).set_lru_capacity(DEFAULT_FILE_TEXT_LRU_CAP); base_db::ParseQuery.in_db_mut(self).set_lru_capacity( lru_capacities .get(stringify!(ParseQuery)) @@ -199,7 +202,7 @@ impl RootDatabase { // base_db::ProcMacrosQuery // SourceDatabaseExt - // base_db::FileTextQuery + base_db::FileTextQuery // base_db::FileSourceRootQuery // base_db::SourceRootQuery base_db::SourceRootCratesQuery |