Unnamed repository; edit this file 'description' to name the repository.
Merge #10025
10025: Don't mutate syntax trees when preparing proc-macro input r=Veykril a=Veykril
Fixes #10013
Co-authored-by: Lukas Wirth <[email protected]>
| -rw-r--r-- | Cargo.lock | 2 | ||||
| -rw-r--r-- | crates/hir_expand/Cargo.toml | 1 | ||||
| -rw-r--r-- | crates/hir_expand/src/db.rs | 35 | ||||
| -rw-r--r-- | crates/hir_expand/src/input.rs | 120 | ||||
| -rw-r--r-- | crates/hir_expand/src/lib.rs | 1 | ||||
| -rw-r--r-- | crates/mbe/Cargo.toml | 1 | ||||
| -rw-r--r-- | crates/mbe/src/lib.rs | 2 | ||||
| -rw-r--r-- | crates/mbe/src/syntax_bridge.rs | 32 | ||||
| -rw-r--r-- | crates/mbe/src/tests.rs | 41 |
9 files changed, 100 insertions, 135 deletions
diff --git a/Cargo.lock b/Cargo.lock index 25f1f1cb7e..90fcb2ac27 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -511,6 +511,7 @@ dependencies = [ "cov-mark", "either", "expect-test", + "itertools", "la-arena", "limit", "log", @@ -887,6 +888,7 @@ name = "mbe" version = "0.0.0" dependencies = [ "cov-mark", + "expect-test", "log", "parser", "profile", diff --git a/crates/hir_expand/Cargo.toml b/crates/hir_expand/Cargo.toml index 4645970266..743e807910 100644 --- a/crates/hir_expand/Cargo.toml +++ b/crates/hir_expand/Cargo.toml @@ -14,6 +14,7 @@ log = "0.4.8" either = "1.5.3" rustc-hash = "1.0.0" la-arena = { version = "0.2.0", path = "../../lib/arena" } +itertools = "0.10.0" base_db = { path = "../base_db", version = "0.0.0" } cfg = { path = "../cfg", version = "0.0.0" } diff --git a/crates/hir_expand/src/db.rs b/crates/hir_expand/src/db.rs index 0c5457016e..fc1dc048ef 100644 --- a/crates/hir_expand/src/db.rs +++ b/crates/hir_expand/src/db.rs @@ -3,19 +3,20 @@ use std::sync::Arc; use base_db::{salsa, SourceDatabase}; +use itertools::Itertools; use limit::Limit; use mbe::{ExpandError, ExpandResult}; use parser::{FragmentKind, T}; use syntax::{ algo::diff, - ast::{self, NameOwner}, - AstNode, GreenNode, Parse, SyntaxNode, SyntaxToken, + ast::{self, AttrsOwner, NameOwner}, + AstNode, GreenNode, Parse, SyntaxNode, SyntaxToken, TextRange, }; use crate::{ - ast_id_map::AstIdMap, hygiene::HygieneFrame, input::process_macro_input, BuiltinAttrExpander, - BuiltinDeriveExpander, BuiltinFnLikeExpander, HirFileId, HirFileIdRepr, MacroCallId, - MacroCallKind, MacroCallLoc, MacroDefId, MacroDefKind, MacroFile, ProcMacroExpander, + ast_id_map::AstIdMap, hygiene::HygieneFrame, BuiltinAttrExpander, BuiltinDeriveExpander, + BuiltinFnLikeExpander, HirFileId, HirFileIdRepr, MacroCallId, MacroCallKind, MacroCallLoc, + MacroDefId, MacroDefKind, MacroFile, ProcMacroExpander, }; /// Total limit on the number of tokens produced by any macro invocation. @@ -257,9 +258,28 @@ fn parse_macro_expansion( fn macro_arg(db: &dyn AstDatabase, id: MacroCallId) -> Option<Arc<(tt::Subtree, mbe::TokenMap)>> { let arg = db.macro_arg_text(id)?; - let (mut tt, tmap) = mbe::syntax_node_to_token_tree(&SyntaxNode::new_root(arg)); + let loc = db.lookup_intern_macro(id); + + let node = SyntaxNode::new_root(arg); + let censor = match loc.kind { + MacroCallKind::FnLike { .. } => None, + MacroCallKind::Derive { derive_attr_index, .. } => match ast::Item::cast(node.clone()) { + Some(item) => item + .attrs() + .map(|attr| attr.syntax().text_range()) + .take(derive_attr_index as usize + 1) + .fold1(TextRange::cover), + None => None, + }, + MacroCallKind::Attr { invoc_attr_index, .. } => match ast::Item::cast(node.clone()) { + Some(item) => { + item.attrs().nth(invoc_attr_index as usize).map(|attr| attr.syntax().text_range()) + } + None => None, + }, + }; + let (mut tt, tmap) = mbe::syntax_node_to_token_tree_censored(&node, censor); - let loc: MacroCallLoc = db.lookup_intern_macro(id); if loc.def.is_proc_macro() { // proc macros expect their inputs without parentheses, MBEs expect it with them included tt.delimiter = None; @@ -271,7 +291,6 @@ fn macro_arg(db: &dyn AstDatabase, id: MacroCallId) -> Option<Arc<(tt::Subtree, fn macro_arg_text(db: &dyn AstDatabase, id: MacroCallId) -> Option<GreenNode> { let loc = db.lookup_intern_macro(id); let arg = loc.kind.arg(db)?; - let arg = process_macro_input(&loc.kind, arg); if matches!(loc.kind, MacroCallKind::FnLike { .. }) { let first = arg.first_child_or_token().map_or(T![.], |it| it.kind()); let last = arg.last_child_or_token().map_or(T![.], |it| it.kind()); diff --git a/crates/hir_expand/src/input.rs b/crates/hir_expand/src/input.rs deleted file mode 100644 index 0ad48a470b..0000000000 --- a/crates/hir_expand/src/input.rs +++ /dev/null @@ -1,120 +0,0 @@ -//! Macro input conditioning. - -use syntax::{ - ast::{self, make, AttrsOwner}, - AstNode, SyntaxNode, -}; - -use crate::{ - name::{name, AsName}, - MacroCallKind, -}; - -pub(crate) fn process_macro_input(macro_call_kind: &MacroCallKind, node: SyntaxNode) -> SyntaxNode { - match macro_call_kind { - MacroCallKind::FnLike { .. } => node, - MacroCallKind::Derive { derive_attr_index, .. } => { - let item = match ast::Item::cast(node.clone()) { - Some(item) => item, - None => return node, - }; - - remove_derives_up_to(item, *derive_attr_index as usize).syntax().clone() - } - MacroCallKind::Attr { invoc_attr_index, .. } => { - let item = match ast::Item::cast(node.clone()) { - Some(item) => item, - None => return node, - }; - - remove_attr_invoc(item, *invoc_attr_index as usize).syntax().clone() - } - } -} - -/// Removes `#[derive]` attributes from `item`, up to `attr_index`. -fn remove_derives_up_to(item: ast::Item, attr_index: usize) -> ast::Item { - let item = item.clone_for_update(); - for attr in item.attrs().take(attr_index + 1) { - if let Some(name) = - attr.path().and_then(|path| path.as_single_segment()).and_then(|seg| seg.name_ref()) - { - if name.as_name() == name![derive] { - replace_attr(&item, &attr); - } - } - } - item -} - -/// Removes the attribute invoking an attribute macro from `item`. -fn remove_attr_invoc(item: ast::Item, attr_index: usize) -> ast::Item { - let item = item.clone_for_update(); - let attr = item - .attrs() - .nth(attr_index) - .unwrap_or_else(|| panic!("cannot find attribute #{}", attr_index)); - replace_attr(&item, &attr); - item -} - -fn replace_attr(item: &ast::Item, attr: &ast::Attr) { - let syntax_index = attr.syntax().index(); - let ws = make::tokens::whitespace(&" ".repeat(u32::from(attr.syntax().text().len()) as usize)); - item.syntax().splice_children(syntax_index..syntax_index + 1, vec![ws.into()]); -} - -#[cfg(test)] -mod tests { - use base_db::{fixture::WithFixture, SourceDatabase}; - use expect_test::{expect, Expect}; - - use crate::test_db::TestDB; - - use super::*; - - fn test_remove_derives_up_to(attr: usize, ra_fixture: &str, expect: Expect) { - let (db, file_id) = TestDB::with_single_file(ra_fixture); - let parsed = db.parse(file_id); - - let mut items: Vec<_> = - parsed.syntax_node().descendants().filter_map(ast::Item::cast).collect(); - assert_eq!(items.len(), 1); - - let item = remove_derives_up_to(items.pop().unwrap(), attr); - let res: String = - item.syntax().children_with_tokens().map(|e| format!("{:?}\n", e)).collect(); - expect.assert_eq(&res); - } - - #[test] - fn remove_derive() { - test_remove_derives_up_to( - 2, - r#" -#[allow(unused)] -#[derive(Copy)] -#[derive(Hello)] -#[derive(Clone)] -struct A { - bar: u32 -} - "#, - expect![[r#" - Node([email protected]) - Token([email protected] "\n") - Token([email protected] " ") - Token([email protected] "\n") - Token([email protected] " ") - Token([email protected] "\n") - Node([email protected]) - Token([email protected] "\n") - Token([email protected] "struct") - Token([email protected] " ") - Node([email protected]) - Token([email protected] " ") - Node([email protected]) - "#]], - ); - } -} diff --git a/crates/hir_expand/src/lib.rs b/crates/hir_expand/src/lib.rs index 40380e1df1..a12dd07426 100644 --- a/crates/hir_expand/src/lib.rs +++ b/crates/hir_expand/src/lib.rs @@ -14,7 +14,6 @@ pub mod builtin_macro; pub mod proc_macro; pub mod quote; pub mod eager; -mod input; use base_db::ProcMacroKind; use either::Either; diff --git a/crates/mbe/Cargo.toml b/crates/mbe/Cargo.toml index 0769c436f6..411bb75dbc 100644 --- a/crates/mbe/Cargo.toml +++ b/crates/mbe/Cargo.toml @@ -13,6 +13,7 @@ cov-mark = "2.0.0-pre.1" rustc-hash = "1.1.0" smallvec = "1.2.0" log = "0.4.8" +expect-test = "1.1" syntax = { path = "../syntax", version = "0.0.0" } parser = { path = "../parser", version = "0.0.0" } diff --git a/crates/mbe/src/lib.rs b/crates/mbe/src/lib.rs index d2b955c5c8..242d07c9ec 100644 --- a/crates/mbe/src/lib.rs +++ b/crates/mbe/src/lib.rs @@ -67,7 +67,7 @@ impl fmt::Display for ExpandError { pub use crate::{ syntax_bridge::{ parse_exprs_with_sep, parse_to_token_tree, syntax_node_to_token_tree, - token_tree_to_syntax_node, + syntax_node_to_token_tree_censored, token_tree_to_syntax_node, }, token_map::TokenMap, }; diff --git a/crates/mbe/src/syntax_bridge.rs b/crates/mbe/src/syntax_bridge.rs index 0421d4c9b0..6aa034abdc 100644 --- a/crates/mbe/src/syntax_bridge.rs +++ b/crates/mbe/src/syntax_bridge.rs @@ -1,5 +1,7 @@ //! Conversions between [`SyntaxNode`] and [`tt::TokenTree`]. +use std::iter; + use parser::{FragmentKind, ParseError, TreeSink}; use rustc_hash::FxHashMap; use syntax::{ @@ -16,8 +18,17 @@ use crate::{ExpandError, TokenMap}; /// Convert the syntax node to a `TokenTree` (what macro /// will consume). pub fn syntax_node_to_token_tree(node: &SyntaxNode) -> (tt::Subtree, TokenMap) { + syntax_node_to_token_tree_censored(node, None) +} + +/// Convert the syntax node to a `TokenTree` (what macro will consume) +/// with the censored range excluded. +pub fn syntax_node_to_token_tree_censored( + node: &SyntaxNode, + censor: Option<TextRange>, +) -> (tt::Subtree, TokenMap) { let global_offset = node.text_range().start(); - let mut c = Convertor::new(node, global_offset); + let mut c = Convertor::new(node, global_offset, censor); let subtree = convert_tokens(&mut c); c.id_alloc.map.shrink_to_fit(); (subtree, c.id_alloc.map) @@ -446,16 +457,24 @@ impl<'a> TokenConvertor for RawConvertor<'a> { struct Convertor { id_alloc: TokenIdAlloc, current: Option<SyntaxToken>, + censor: Option<TextRange>, range: TextRange, punct_offset: Option<(SyntaxToken, TextSize)>, } impl Convertor { - fn new(node: &SyntaxNode, global_offset: TextSize) -> Convertor { + fn new(node: &SyntaxNode, global_offset: TextSize, censor: Option<TextRange>) -> Convertor { + let first = node.first_token(); + let current = match censor { + Some(censor) => iter::successors(first, |token| token.next_token()) + .find(|token| !censor.contains_range(token.text_range())), + None => first, + }; Convertor { id_alloc: { TokenIdAlloc { map: TokenMap::default(), global_offset, next_id: 0 } }, - current: node.first_token(), + current, range: node.text_range(), + censor, punct_offset: None, } } @@ -512,8 +531,11 @@ impl TokenConvertor for Convertor { if !&self.range.contains_range(curr.text_range()) { return None; } - self.current = curr.next_token(); - + self.current = match self.censor { + Some(censor) => iter::successors(curr.next_token(), |token| token.next_token()) + .find(|token| !censor.contains_range(token.text_range())), + None => curr.next_token(), + }; let token = if curr.kind().is_punct() { let range = curr.text_range(); let range = TextRange::at(range.start(), TextSize::of('.')); diff --git a/crates/mbe/src/tests.rs b/crates/mbe/src/tests.rs index 705cf5a2b1..c2a1696b3d 100644 --- a/crates/mbe/src/tests.rs +++ b/crates/mbe/src/tests.rs @@ -228,3 +228,44 @@ fn debug_dump_ignore_spaces(node: &syntax::SyntaxNode) -> String { buf } + +#[test] +fn test_node_to_tt_censor() { + use syntax::ast::{AttrsOwner, ModuleItemOwner}; + + let source = r##" +#[attr0] +#[attr1] +#[attr2] +struct Struct { + field: () +} +"##; + let source_file = ast::SourceFile::parse(&source).ok().unwrap(); + let item = source_file.items().next().unwrap(); + let attr = item.attrs().nth(1).unwrap(); + + let (tt, _) = + syntax_node_to_token_tree_censored(item.syntax(), Some(attr.syntax().text_range())); + expect_test::expect![[r##"# [attr0] # [attr2] struct Struct {field : ()}"##]] + .assert_eq(&tt.to_string()); + + let source = r##" +#[derive(Derive0)] +#[derive(Derive1)] +#[derive(Derive2)] +struct Struct { + field: () +} +"##; + let source_file = ast::SourceFile::parse(&source).ok().unwrap(); + let item = source_file.items().next().unwrap(); + let attr = item.attrs().nth(1).unwrap(); + + let (tt, _) = syntax_node_to_token_tree_censored( + item.syntax(), + Some(attr.syntax().text_range().cover_offset(0.into())), + ); + expect_test::expect![[r##"# [derive (Derive2)] struct Struct {field : ()}"##]] + .assert_eq(&tt.to_string()); +} |