Unnamed repository; edit this file 'description' to name the repository.
Implement rpc protocol changes
| -rw-r--r-- | Cargo.lock | 14 | ||||
| -rw-r--r-- | crates/proc-macro-api/Cargo.toml | 4 | ||||
| -rw-r--r-- | crates/proc-macro-api/src/legacy_protocol.rs | 5 | ||||
| -rw-r--r-- | crates/proc-macro-api/src/legacy_protocol/msg.rs | 2 | ||||
| -rw-r--r-- | crates/proc-macro-api/src/legacy_protocol/msg/flat.rs | 358 | ||||
| -rw-r--r-- | crates/proc-macro-api/src/lib.rs | 7 | ||||
| -rw-r--r-- | crates/proc-macro-srv-cli/Cargo.toml | 4 | ||||
| -rw-r--r-- | crates/proc-macro-srv-cli/src/main_loop.rs | 23 | ||||
| -rw-r--r-- | crates/proc-macro-srv/src/bridge.rs | 12 | ||||
| -rw-r--r-- | crates/proc-macro-srv/src/lib.rs | 6 | ||||
| -rw-r--r-- | crates/proc-macro-srv/src/server_impl.rs | 2 | ||||
| -rw-r--r-- | crates/proc-macro-srv/src/token_stream.rs | 33 | ||||
| -rw-r--r-- | xtask/src/install.rs | 13 |
13 files changed, 401 insertions, 82 deletions
diff --git a/Cargo.lock b/Cargo.lock index 22d41fc304..caab97979f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -772,7 +772,6 @@ dependencies = [ "hir-def", "hir-expand", "hir-ty", - "indexmap", "intern", "itertools 0.14.0", "ra-ap-rustc_type_ir", @@ -824,7 +823,6 @@ dependencies = [ "syntax-bridge", "test-fixture", "test-utils", - "text-size 1.1.1 (registry+https://github.com/rust-lang/crates.io-index)", "thin-vec", "tracing", "triomphe", @@ -864,7 +862,6 @@ version = "0.0.0" dependencies = [ "arrayvec", "base-db", - "bitflags 2.9.4", "cov-mark", "either", "ena", @@ -890,7 +887,6 @@ dependencies = [ "rustc_apfloat", "salsa", "salsa-macros", - "scoped-tls", "smallvec", "span", "stdx", @@ -1085,7 +1081,6 @@ dependencies = [ "expect-test", "fst", "hir", - "indexmap", "itertools 0.14.0", "line-index 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", "macros", @@ -1825,6 +1820,7 @@ dependencies = [ "indexmap", "intern", "paths", + "proc-macro-srv", "rustc-hash 2.1.1", "serde", "serde_derive", @@ -2289,14 +2285,12 @@ dependencies = [ "ide-db", "ide-ssr", "indexmap", - "intern", "itertools 0.14.0", "load-cargo", "lsp-server 0.7.9 (registry+https://github.com/rust-lang/crates.io-index)", "lsp-types", "memchr", "mimalloc", - "nohash-hasher", "num_cpus", "oorandom", "parking_lot", @@ -2490,12 +2484,6 @@ dependencies = [ ] [[package]] -name = "scoped-tls" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1cf6437eb19a8f4a6cc0f7dca544973b0b78843adbfeb3683d1a94a0024a294" - -[[package]] name = "scopeguard" version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" diff --git a/crates/proc-macro-api/Cargo.toml b/crates/proc-macro-api/Cargo.toml index dac8e09435..63745b9f74 100644 --- a/crates/proc-macro-api/Cargo.toml +++ b/crates/proc-macro-api/Cargo.toml @@ -24,10 +24,14 @@ indexmap.workspace = true paths = { workspace = true, features = ["serde1"] } tt.workspace = true stdx.workspace = true +proc-macro-srv = {workspace = true, optional = true} # span = {workspace = true, default-features = false} does not work span = { path = "../span", version = "0.0.0", default-features = false} intern.workspace = true +[features] +sysroot-abi = ["proc-macro-srv", "proc-macro-srv/sysroot-abi"] + [lints] workspace = true diff --git a/crates/proc-macro-api/src/legacy_protocol.rs b/crates/proc-macro-api/src/legacy_protocol.rs index ee96b899fe..0a72052cc5 100644 --- a/crates/proc-macro-api/src/legacy_protocol.rs +++ b/crates/proc-macro-api/src/legacy_protocol.rs @@ -95,9 +95,10 @@ pub(crate) fn expand( let mixed_site = span_data_table.insert_full(mixed_site).0; let task = ExpandMacro { data: ExpandMacroData { - macro_body: FlatTree::new(subtree, version, &mut span_data_table), + macro_body: FlatTree::from_subtree(subtree, version, &mut span_data_table), macro_name: proc_macro.name.to_string(), - attributes: attr.map(|subtree| FlatTree::new(subtree, version, &mut span_data_table)), + attributes: attr + .map(|subtree| FlatTree::from_subtree(subtree, version, &mut span_data_table)), has_global_spans: ExpnGlobals { serialize: version >= version::HAS_GLOBAL_SPANS, def_site, diff --git a/crates/proc-macro-api/src/legacy_protocol/msg.rs b/crates/proc-macro-api/src/legacy_protocol/msg.rs index b795c45589..487f50b145 100644 --- a/crates/proc-macro-api/src/legacy_protocol/msg.rs +++ b/crates/proc-macro-api/src/legacy_protocol/msg.rs @@ -297,7 +297,7 @@ mod tests { let mut span_data_table = Default::default(); let task = ExpandMacro { data: ExpandMacroData { - macro_body: FlatTree::new(tt.view(), v, &mut span_data_table), + macro_body: FlatTree::from_subtree(tt.view(), v, &mut span_data_table), macro_name: Default::default(), attributes: None, has_global_spans: ExpnGlobals { diff --git a/crates/proc-macro-api/src/legacy_protocol/msg/flat.rs b/crates/proc-macro-api/src/legacy_protocol/msg/flat.rs index fb3542d24f..af3ea76861 100644 --- a/crates/proc-macro-api/src/legacy_protocol/msg/flat.rs +++ b/crates/proc-macro-api/src/legacy_protocol/msg/flat.rs @@ -8,8 +8,7 @@ //! about performance here a bit. //! //! So what this module does is dumping a `tt::TopSubtree` into a bunch of flat -//! array of numbers. See the test in the parent module to get an example -//! output. +//! array of numbers. //! //! ```json //! { @@ -38,6 +37,7 @@ use std::collections::VecDeque; use intern::Symbol; +use proc_macro_srv::TokenStream; use rustc_hash::FxHashMap; use serde_derive::{Deserialize, Serialize}; use span::{EditionedFileId, ErasedFileAstId, Span, SpanAnchor, SyntaxContext, TextRange}; @@ -120,12 +120,12 @@ struct IdentRepr { } impl FlatTree { - pub fn new( + pub fn from_subtree( subtree: tt::SubtreeView<'_, Span>, version: u32, span_data_table: &mut SpanDataIndexMap, ) -> FlatTree { - let mut w = Writer::<Span> { + let mut w = Writer::<Span, _> { string_table: FxHashMap::default(), work: VecDeque::new(), span_data_table, @@ -138,7 +138,7 @@ impl FlatTree { text: Vec::new(), version, }; - w.write(subtree); + w.write_subtree(subtree); FlatTree { subtree: if version >= ENCODE_CLOSE_SPAN_VERSION { @@ -162,11 +162,64 @@ impl FlatTree { } } - pub fn new_raw<T: SpanTransformer<Table = ()>>( - subtree: tt::SubtreeView<'_, T::Span>, + pub fn from_tokenstream( + tokenstream: proc_macro_srv::TokenStream<Span>, + version: u32, + call_site: Span, + span_data_table: &mut SpanDataIndexMap, + ) -> FlatTree { + let mut w = Writer::<Span, _> { + string_table: FxHashMap::default(), + work: VecDeque::new(), + span_data_table, + + subtree: Vec::new(), + literal: Vec::new(), + punct: Vec::new(), + ident: Vec::new(), + token_tree: Vec::new(), + text: Vec::new(), + version, + }; + let group = proc_macro_srv::Group { + delimiter: proc_macro_srv::Delimiter::None, + stream: Some(tokenstream), + span: proc_macro_srv::DelimSpan { + open: call_site, + close: call_site, + entire: call_site, + }, + }; + w.write_tokenstream(&group); + + FlatTree { + subtree: if version >= ENCODE_CLOSE_SPAN_VERSION { + write_vec(w.subtree, SubtreeRepr::write_with_close_span) + } else { + write_vec(w.subtree, SubtreeRepr::write) + }, + literal: if version >= EXTENDED_LEAF_DATA { + write_vec(w.literal, LiteralRepr::write_with_kind) + } else { + write_vec(w.literal, LiteralRepr::write) + }, + punct: write_vec(w.punct, PunctRepr::write), + ident: if version >= EXTENDED_LEAF_DATA { + write_vec(w.ident, IdentRepr::write_with_rawness) + } else { + write_vec(w.ident, IdentRepr::write) + }, + token_tree: w.token_tree, + text: w.text, + } + } + + pub fn from_tokenstream_raw<T: SpanTransformer<Table = ()>>( + tokenstream: proc_macro_srv::TokenStream<T::Span>, + call_site: T::Span, version: u32, ) -> FlatTree { - let mut w = Writer::<T> { + let mut w = Writer::<T, _> { string_table: FxHashMap::default(), work: VecDeque::new(), span_data_table: &mut (), @@ -179,7 +232,16 @@ impl FlatTree { text: Vec::new(), version, }; - w.write(subtree); + let group = proc_macro_srv::Group { + delimiter: proc_macro_srv::Delimiter::None, + stream: Some(tokenstream), + span: proc_macro_srv::DelimSpan { + open: call_site, + close: call_site, + entire: call_site, + }, + }; + w.write_tokenstream(&group); FlatTree { subtree: if version >= ENCODE_CLOSE_SPAN_VERSION { @@ -230,13 +292,14 @@ impl FlatTree { span_data_table, version, } - .read() + .read_subtree() } - - pub fn to_subtree_unresolved<T: SpanTransformer<Table = ()>>( +} +impl FlatTree { + pub fn to_tokenstream_unresolved<T: SpanTransformer<Table = ()>>( self, version: u32, - ) -> tt::TopSubtree<T::Span> { + ) -> proc_macro_srv::TokenStream<T::Span> { Reader::<T> { subtree: if version >= ENCODE_CLOSE_SPAN_VERSION { read_vec(self.subtree, SubtreeRepr::read_with_close_span) @@ -259,7 +322,37 @@ impl FlatTree { span_data_table: &(), version, } - .read() + .read_tokenstream() + } + + pub fn to_tokenstream_resolved( + self, + version: u32, + span_data_table: &SpanDataIndexMap, + ) -> proc_macro_srv::TokenStream<Span> { + Reader::<Span> { + subtree: if version >= ENCODE_CLOSE_SPAN_VERSION { + read_vec(self.subtree, SubtreeRepr::read_with_close_span) + } else { + read_vec(self.subtree, SubtreeRepr::read) + }, + literal: if version >= EXTENDED_LEAF_DATA { + read_vec(self.literal, LiteralRepr::read_with_kind) + } else { + read_vec(self.literal, LiteralRepr::read) + }, + punct: read_vec(self.punct, PunctRepr::read), + ident: if version >= EXTENDED_LEAF_DATA { + read_vec(self.ident, IdentRepr::read_with_rawness) + } else { + read_vec(self.ident, IdentRepr::read) + }, + token_tree: self.token_tree, + text: self.text, + span_data_table, + version, + } + .read_tokenstream() } } @@ -391,8 +484,8 @@ impl SpanTransformer for Span { } } -struct Writer<'a, 'span, S: SpanTransformer> { - work: VecDeque<(usize, tt::iter::TtIter<'a, S::Span>)>, +struct Writer<'a, 'span, S: SpanTransformer, W> { + work: VecDeque<(usize, W)>, string_table: FxHashMap<std::borrow::Cow<'a, str>, u32>, span_data_table: &'span mut S::Table, version: u32, @@ -405,8 +498,8 @@ struct Writer<'a, 'span, S: SpanTransformer> { text: Vec<String>, } -impl<'a, T: SpanTransformer> Writer<'a, '_, T> { - fn write(&mut self, root: tt::SubtreeView<'a, T::Span>) { +impl<'a, T: SpanTransformer> Writer<'a, '_, T, tt::iter::TtIter<'a, T::Span>> { + fn write_subtree(&mut self, root: tt::SubtreeView<'a, T::Span>) { let subtree = root.top_subtree(); self.enqueue(subtree, root.iter()); while let Some((idx, subtree)) = self.work.pop_front() { @@ -414,10 +507,6 @@ impl<'a, T: SpanTransformer> Writer<'a, '_, T> { } } - fn token_id_of(&mut self, span: T::Span) -> SpanId { - T::token_id_of(self.span_data_table, span) - } - fn subtree(&mut self, idx: usize, subtree: tt::iter::TtIter<'a, T::Span>) { let mut first_tt = self.token_tree.len(); let n_tt = subtree.clone().count(); // FIXME: `count()` walks over the entire iterator. @@ -502,6 +591,12 @@ impl<'a, T: SpanTransformer> Writer<'a, '_, T> { self.work.push_back((idx, contents)); idx as u32 } +} + +impl<'a, T: SpanTransformer, U> Writer<'a, '_, T, U> { + fn token_id_of(&mut self, span: T::Span) -> SpanId { + T::token_id_of(self.span_data_table, span) + } pub(crate) fn intern(&mut self, text: &'a str) -> u32 { let table = &mut self.text; @@ -522,6 +617,105 @@ impl<'a, T: SpanTransformer> Writer<'a, '_, T> { } } +#[cfg(feature = "sysroot-abi")] +impl<'a, T: SpanTransformer> Writer<'a, '_, T, &'a proc_macro_srv::Group<T::Span>> { + fn write_tokenstream(&mut self, root: &'a proc_macro_srv::Group<T::Span>) { + self.enqueue_group(root); + + while let Some((idx, group)) = self.work.pop_front() { + self.group(idx, group); + } + } + + fn group(&mut self, idx: usize, group: &'a proc_macro_srv::Group<T::Span>) { + let mut first_tt = self.token_tree.len(); + let n_tt = group.stream.as_ref().map_or(0, |it| it.len()); + self.token_tree.resize(first_tt + n_tt, !0); + + self.subtree[idx].tt = [first_tt as u32, (first_tt + n_tt) as u32]; + + for tt in group.stream.iter().flat_map(|it| it.iter()) { + let idx_tag = match tt { + proc_macro_srv::TokenTree::Group(group) => { + let idx = self.enqueue_group(group); + idx << 2 + } + proc_macro_srv::TokenTree::Literal(lit) => { + let idx = self.literal.len() as u32; + let id = self.token_id_of(lit.span); + let (text, suffix) = if self.version >= EXTENDED_LEAF_DATA { + ( + self.intern(lit.symbol.as_str()), + lit.suffix.as_ref().map(|s| self.intern(s.as_str())).unwrap_or(!0), + ) + } else { + (self.intern_owned(proc_macro_srv::literal_to_string(lit)), !0) + }; + self.literal.push(LiteralRepr { + id, + text, + kind: u16::from_le_bytes(match lit.kind { + proc_macro_srv::LitKind::ErrWithGuar => [0, 0], + proc_macro_srv::LitKind::Byte => [1, 0], + proc_macro_srv::LitKind::Char => [2, 0], + proc_macro_srv::LitKind::Integer => [3, 0], + proc_macro_srv::LitKind::Float => [4, 0], + proc_macro_srv::LitKind::Str => [5, 0], + proc_macro_srv::LitKind::StrRaw(r) => [6, r], + proc_macro_srv::LitKind::ByteStr => [7, 0], + proc_macro_srv::LitKind::ByteStrRaw(r) => [8, r], + proc_macro_srv::LitKind::CStr => [9, 0], + proc_macro_srv::LitKind::CStrRaw(r) => [10, r], + }), + suffix, + }); + (idx << 2) | 0b01 + } + proc_macro_srv::TokenTree::Punct(punct) => { + let idx = self.punct.len() as u32; + let id = self.token_id_of(punct.span); + self.punct.push(PunctRepr { + char: punct.ch as char, + spacing: if punct.joint { tt::Spacing::Joint } else { tt::Spacing::Alone }, + id, + }); + (idx << 2) | 0b10 + } + proc_macro_srv::TokenTree::Ident(ident) => { + let idx = self.ident.len() as u32; + let id = self.token_id_of(ident.span); + let text = if self.version >= EXTENDED_LEAF_DATA { + self.intern(ident.sym.as_str()) + } else if ident.is_raw { + self.intern_owned(format!("r#{}", ident.sym.as_str(),)) + } else { + self.intern(ident.sym.as_str()) + }; + self.ident.push(IdentRepr { id, text, is_raw: ident.is_raw }); + (idx << 2) | 0b11 + } + }; + self.token_tree[first_tt] = idx_tag; + first_tt += 1; + } + } + + fn enqueue_group(&mut self, group: &'a proc_macro_srv::Group<T::Span>) -> u32 { + let idx = self.subtree.len(); + let open = self.token_id_of(group.span.open); + let close = self.token_id_of(group.span.close); + let delimiter_kind = match group.delimiter { + proc_macro_srv::Delimiter::Parenthesis => tt::DelimiterKind::Parenthesis, + proc_macro_srv::Delimiter::Brace => tt::DelimiterKind::Brace, + proc_macro_srv::Delimiter::Bracket => tt::DelimiterKind::Bracket, + proc_macro_srv::Delimiter::None => tt::DelimiterKind::Invisible, + }; + self.subtree.push(SubtreeRepr { open, close, kind: delimiter_kind, tt: [!0, !0] }); + self.work.push_back((idx, group)); + idx as u32 + } +} + struct Reader<'span, S: SpanTransformer> { version: u32, subtree: Vec<SubtreeRepr>, @@ -534,7 +728,7 @@ struct Reader<'span, S: SpanTransformer> { } impl<T: SpanTransformer> Reader<'_, T> { - pub(crate) fn read(self) -> tt::TopSubtree<T::Span> { + pub(crate) fn read_subtree(self) -> tt::TopSubtree<T::Span> { let mut res: Vec<Option<(tt::Delimiter<T::Span>, Vec<tt::TokenTree<T::Span>>)>> = vec![None; self.subtree.len()]; let read_span = |id| T::span_for_token_id(self.span_data_table, id); @@ -641,3 +835,121 @@ impl<T: SpanTransformer> Reader<'_, T> { tt::TopSubtree(res.into_boxed_slice()) } } + +impl<T: SpanTransformer> Reader<'_, T> { + pub(crate) fn read_tokenstream(self) -> proc_macro_srv::TokenStream<T::Span> { + let mut res: Vec<Option<proc_macro_srv::Group<T::Span>>> = vec![None; self.subtree.len()]; + let read_span = |id| T::span_for_token_id(self.span_data_table, id); + for i in (0..self.subtree.len()).rev() { + let repr = &self.subtree[i]; + let token_trees = &self.token_tree[repr.tt[0] as usize..repr.tt[1] as usize]; + + let stream = token_trees + .iter() + .copied() + .map(|idx_tag| { + let tag = idx_tag & 0b11; + let idx = (idx_tag >> 2) as usize; + match tag { + // XXX: we iterate subtrees in reverse to guarantee + // that this unwrap doesn't fire. + 0b00 => proc_macro_srv::TokenTree::Group(res[idx].take().unwrap()), + 0b01 => { + let repr = &self.literal[idx]; + let text = self.text[repr.text as usize].as_str(); + let span = read_span(repr.id); + proc_macro_srv::TokenTree::Literal( + if self.version >= EXTENDED_LEAF_DATA { + proc_macro_srv::Literal { + symbol: Symbol::intern(text), + span, + kind: match u16::to_le_bytes(repr.kind) { + [0, _] => proc_macro_srv::LitKind::ErrWithGuar, + [1, _] => proc_macro_srv::LitKind::Byte, + [2, _] => proc_macro_srv::LitKind::Char, + [3, _] => proc_macro_srv::LitKind::Integer, + [4, _] => proc_macro_srv::LitKind::Float, + [5, _] => proc_macro_srv::LitKind::Str, + [6, r] => proc_macro_srv::LitKind::StrRaw(r), + [7, _] => proc_macro_srv::LitKind::ByteStr, + [8, r] => proc_macro_srv::LitKind::ByteStrRaw(r), + [9, _] => proc_macro_srv::LitKind::CStr, + [10, r] => proc_macro_srv::LitKind::CStrRaw(r), + _ => unreachable!(), + }, + suffix: if repr.suffix != !0 { + Some(Symbol::intern( + self.text[repr.suffix as usize].as_str(), + )) + } else { + None + }, + } + } else { + proc_macro_srv::literal_from_str(text, span).unwrap_or_else( + |_| proc_macro_srv::Literal { + symbol: Symbol::intern("internal error"), + span, + kind: proc_macro_srv::LitKind::ErrWithGuar, + suffix: None, + }, + ) + }, + ) + } + 0b10 => { + let repr = &self.punct[idx]; + proc_macro_srv::TokenTree::Punct(proc_macro_srv::Punct { + ch: repr.char as u8, + joint: repr.spacing == tt::Spacing::Joint, + span: read_span(repr.id), + }) + } + 0b11 => { + let repr = &self.ident[idx]; + let text = self.text[repr.text as usize].as_str(); + let (is_raw, text) = if self.version >= EXTENDED_LEAF_DATA { + ( + if repr.is_raw { + tt::IdentIsRaw::Yes + } else { + tt::IdentIsRaw::No + }, + text, + ) + } else { + tt::IdentIsRaw::split_from_symbol(text) + }; + proc_macro_srv::TokenTree::Ident(proc_macro_srv::Ident { + sym: Symbol::intern(text), + span: read_span(repr.id), + is_raw: is_raw.yes(), + }) + } + other => panic!("bad tag: {other}"), + } + }) + .collect::<Vec<_>>(); + let g = proc_macro_srv::Group { + delimiter: match repr.kind { + tt::DelimiterKind::Parenthesis => proc_macro_srv::Delimiter::Parenthesis, + tt::DelimiterKind::Brace => proc_macro_srv::Delimiter::Brace, + tt::DelimiterKind::Bracket => proc_macro_srv::Delimiter::Bracket, + tt::DelimiterKind::Invisible => proc_macro_srv::Delimiter::None, + }, + stream: if stream.is_empty() { None } else { Some(TokenStream::new(stream)) }, + span: proc_macro_srv::DelimSpan { + open: read_span(repr.open), + close: read_span(repr.close), + // FIXME + entire: read_span(repr.close), + }, + }; + res[i] = Some(g); + } + // FIXME: double check this + proc_macro_srv::TokenStream::new(vec![proc_macro_srv::TokenTree::Group( + res[0].take().unwrap(), + )]) + } +} diff --git a/crates/proc-macro-api/src/lib.rs b/crates/proc-macro-api/src/lib.rs index 97919b85b5..870d81f976 100644 --- a/crates/proc-macro-api/src/lib.rs +++ b/crates/proc-macro-api/src/lib.rs @@ -5,6 +5,13 @@ //! is used to provide basic infrastructure for communication between two //! processes: Client (RA itself), Server (the external program) +#![cfg_attr(not(feature = "sysroot-abi"), allow(unused_crate_dependencies))] +#![cfg_attr( + feature = "sysroot-abi", + feature(proc_macro_internals, proc_macro_diagnostic, proc_macro_span) +)] +#![allow(internal_features)] + pub mod legacy_protocol; mod process; diff --git a/crates/proc-macro-srv-cli/Cargo.toml b/crates/proc-macro-srv-cli/Cargo.toml index 91e9e62b08..38c42729d6 100644 --- a/crates/proc-macro-srv-cli/Cargo.toml +++ b/crates/proc-macro-srv-cli/Cargo.toml @@ -18,8 +18,8 @@ clap = {version = "4.5.42", default-features = false, features = ["std"]} postcard = { version = "1.1.3", optional = true } [features] -default = ["postcard"] -sysroot-abi = ["proc-macro-srv/sysroot-abi"] +default = ["postcard", "sysroot-abi"] +sysroot-abi = ["proc-macro-srv/sysroot-abi", "proc-macro-api/sysroot-abi"] in-rust-tree = ["proc-macro-srv/in-rust-tree", "sysroot-abi"] postcard = ["dep:postcard"] diff --git a/crates/proc-macro-srv-cli/src/main_loop.rs b/crates/proc-macro-srv-cli/src/main_loop.rs index 703bc965db..5533107570 100644 --- a/crates/proc-macro-srv-cli/src/main_loop.rs +++ b/crates/proc-macro-srv-cli/src/main_loop.rs @@ -91,9 +91,10 @@ fn run_json() -> io::Result<()> { let mixed_site = SpanId(mixed_site as u32); let macro_body = - macro_body.to_subtree_unresolved::<SpanTrans>(CURRENT_API_VERSION); - let attributes = attributes - .map(|it| it.to_subtree_unresolved::<SpanTrans>(CURRENT_API_VERSION)); + macro_body.to_tokenstream_unresolved::<SpanTrans>(CURRENT_API_VERSION); + let attributes = attributes.map(|it| { + it.to_tokenstream_unresolved::<SpanTrans>(CURRENT_API_VERSION) + }); srv.expand( lib, @@ -107,8 +108,9 @@ fn run_json() -> io::Result<()> { mixed_site, ) .map(|it| { - msg::FlatTree::new_raw::<SpanTrans>( - tt::SubtreeView::new(&it), + msg::FlatTree::from_tokenstream_raw::<SpanTrans>( + it, + call_site, CURRENT_API_VERSION, ) }) @@ -122,10 +124,10 @@ fn run_json() -> io::Result<()> { let call_site = span_data_table[call_site]; let mixed_site = span_data_table[mixed_site]; - let macro_body = - macro_body.to_subtree_resolved(CURRENT_API_VERSION, &span_data_table); + let macro_body = macro_body + .to_tokenstream_resolved(CURRENT_API_VERSION, &span_data_table); let attributes = attributes.map(|it| { - it.to_subtree_resolved(CURRENT_API_VERSION, &span_data_table) + it.to_tokenstream_resolved(CURRENT_API_VERSION, &span_data_table) }); srv.expand( lib, @@ -140,9 +142,10 @@ fn run_json() -> io::Result<()> { ) .map(|it| { ( - msg::FlatTree::new( - tt::SubtreeView::new(&it), + msg::FlatTree::from_tokenstream( + it, CURRENT_API_VERSION, + call_site, &mut span_data_table, ), serialize_span_data_index_map(&span_data_table), diff --git a/crates/proc-macro-srv/src/bridge.rs b/crates/proc-macro-srv/src/bridge.rs index b6c4692319..71739f3f36 100644 --- a/crates/proc-macro-srv/src/bridge.rs +++ b/crates/proc-macro-srv/src/bridge.rs @@ -1,10 +1,10 @@ use proc_macro::bridge as pm_bridge;
-pub(crate) use pm_bridge::{DelimSpan, Diagnostic, ExpnGlobals, LitKind};
+pub use pm_bridge::{DelimSpan, Diagnostic, ExpnGlobals, LitKind};
-pub(crate) type TokenTree<S> =
+pub type TokenTree<S> =
pm_bridge::TokenTree<crate::token_stream::TokenStream<S>, S, intern::Symbol>;
-pub(crate) type Literal<S> = pm_bridge::Literal<S, intern::Symbol>;
-pub(crate) type Group<S> = pm_bridge::Group<crate::token_stream::TokenStream<S>, S>;
-pub(crate) type Punct<S> = pm_bridge::Punct<S>;
-pub(crate) type Ident<S> = pm_bridge::Ident<S, intern::Symbol>;
+pub type Literal<S> = pm_bridge::Literal<S, intern::Symbol>;
+pub type Group<S> = pm_bridge::Group<crate::token_stream::TokenStream<S>, S>;
+pub type Punct<S> = pm_bridge::Punct<S>;
+pub type Ident<S> = pm_bridge::Ident<S, intern::Symbol>;
diff --git a/crates/proc-macro-srv/src/lib.rs b/crates/proc-macro-srv/src/lib.rs index cf125cbec6..1f80e93db6 100644 --- a/crates/proc-macro-srv/src/lib.rs +++ b/crates/proc-macro-srv/src/lib.rs @@ -47,6 +47,12 @@ use temp_dir::TempDir; pub use crate::server_impl::token_id::SpanId; +pub use proc_macro::Delimiter; + +pub use crate::bridge::*; +pub use crate::server_impl::literal_from_str; +pub use crate::token_stream::{TokenStream, literal_to_string}; + #[derive(Copy, Clone, Eq, PartialEq, Debug)] pub enum ProcMacroKind { CustomDerive, diff --git a/crates/proc-macro-srv/src/server_impl.rs b/crates/proc-macro-srv/src/server_impl.rs index bc46f8f0e4..bacead1a88 100644 --- a/crates/proc-macro-srv/src/server_impl.rs +++ b/crates/proc-macro-srv/src/server_impl.rs @@ -9,7 +9,7 @@ pub(crate) mod rust_analyzer_span; pub(crate) mod token_id; -pub(super) fn literal_from_str<Span: Copy>( +pub fn literal_from_str<Span: Copy>( s: &str, span: Span, ) -> Result<crate::bridge::Literal<Span>, ()> { diff --git a/crates/proc-macro-srv/src/token_stream.rs b/crates/proc-macro-srv/src/token_stream.rs index 9a8d0cea8f..e337f08ce4 100644 --- a/crates/proc-macro-srv/src/token_stream.rs +++ b/crates/proc-macro-srv/src/token_stream.rs @@ -22,30 +22,22 @@ impl<S> Default for TokenStream<S> { }
impl<S> TokenStream<S> {
- pub(crate) fn new(tts: Vec<TokenTree<S>>) -> TokenStream<S> {
+ pub fn new(tts: Vec<TokenTree<S>>) -> TokenStream<S> {
TokenStream(Arc::new(tts))
}
- pub(crate) fn is_empty(&self) -> bool {
+ pub fn is_empty(&self) -> bool {
self.0.is_empty()
}
- pub(crate) fn len(&self) -> usize {
+ pub fn len(&self) -> usize {
self.0.len()
}
- pub(crate) fn get(&self, index: usize) -> Option<&TokenTree<S>> {
- self.0.get(index)
- }
-
- pub(crate) fn iter(&self) -> TokenStreamIter<'_, S> {
+ pub fn iter(&self) -> TokenStreamIter<'_, S> {
TokenStreamIter::new(self)
}
- pub(crate) fn chunks(&self, chunk_size: usize) -> core::slice::Chunks<'_, TokenTree<S>> {
- self.0.chunks(chunk_size)
- }
-
pub(crate) fn from_str(s: &str, span: S) -> Result<Self, String>
where
S: SpanLike + Copy,
@@ -481,7 +473,13 @@ fn display_token_tree<S>(tt: &TokenTree<S>, f: &mut std::fmt::Formatter<'_>) -> Ok(())
}
-fn display_fmt_literal<S>(literal: &Literal<S>, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+pub fn literal_to_string<S>(literal: &Literal<S>) -> String {
+ let mut buf = String::new();
+ display_fmt_literal(literal, &mut buf).unwrap();
+ buf
+}
+
+fn display_fmt_literal<S>(literal: &Literal<S>, f: &mut impl std::fmt::Write) -> fmt::Result {
match literal.kind {
LitKind::Byte => write!(f, "b'{}'", literal.symbol),
LitKind::Char => write!(f, "'{}'", literal.symbol),
@@ -626,7 +624,7 @@ impl<S> FromIterator<TokenTree<S>> for TokenStream<S> { }
#[derive(Clone)]
-pub(crate) struct TokenStreamIter<'t, S> {
+pub struct TokenStreamIter<'t, S> {
stream: &'t TokenStream<S>,
index: usize,
}
@@ -635,13 +633,6 @@ impl<'t, S> TokenStreamIter<'t, S> { fn new(stream: &'t TokenStream<S>) -> Self {
TokenStreamIter { stream, index: 0 }
}
-
- // Peeking could be done via `Peekable`, but most iterators need peeking,
- // and this is simple and avoids the need to use `peekable` and `Peekable`
- // at all the use sites.
- pub(crate) fn peek(&self) -> Option<&'t TokenTree<S>> {
- self.stream.0.get(self.index)
- }
}
impl<'t, S> Iterator for TokenStreamIter<'t, S> {
diff --git a/xtask/src/install.rs b/xtask/src/install.rs index 975e361ba5..bddce0f027 100644 --- a/xtask/src/install.rs +++ b/xtask/src/install.rs @@ -174,10 +174,17 @@ fn install_server(sh: &Shell, opts: ServerOpt) -> anyhow::Result<()> { fn install_proc_macro_server(sh: &Shell, opts: ProcMacroServerOpt) -> anyhow::Result<()> { let profile = if opts.dev_rel { "dev-rel" } else { "release" }; - cmd!( + let mut cmd = cmd!( sh, - "cargo +nightly install --path crates/proc-macro-srv-cli --profile={profile} --locked --force --features sysroot-abi" - ).run()?; + "cargo install --path crates/proc-macro-srv-cli --profile={profile} --locked --force --features sysroot-abi" + ); + if std::env::var_os("RUSTUP_TOOLCHAIN").is_none() { + cmd = cmd.env("RUSTUP_TOOLCHAIN", "nightly"); + } else { + cmd = cmd.env("RUSTC_BOOTSTRAP", "1"); + } + + cmd.run()?; Ok(()) } |