Unnamed repository; edit this file 'description' to name the repository.
internal: replace L_DOLLAR/R_DOLLAR with parenthesis hack
The general problem we are dealing with here is this:
```
macro_rules! thrice {
($e:expr) => { $e * 3}
}
fn main() {
let x = thrice!(1 + 2);
}
```
we really want this to print 9 rather than 7.
The way rustc solves this is rather ad-hoc. In rustc, token trees are
allowed to include whole AST fragments, so 1+2 is passed through macro
expansion as a single unit. This is a significant violation of token
tree model.
In rust-analyzer, we intended to handle this in a more elegant way,
using token trees with "invisible" delimiters. The idea was is that we
introduce a new kind of parenthesis, "left $"/"right $", and let the
parser intelligently handle this.
The idea was inspired by the relevant comment in the proc_macro crate:
https://doc.rust-lang.org/stable/proc_macro/enum.Delimiter.html#variant.None
> An implicit delimiter, that may, for example, appear around tokens
> coming from a “macro variable” $var. It is important to preserve
> operator priorities in cases like $var * 3 where $var is 1 + 2.
> Implicit delimiters might not survive roundtrip of a token stream
> through a string.
Now that we are older and wiser, we conclude that the idea doesn't work.
_First_, the comment in the proc-macro crate is wishful thinking. Rustc
currently completely ignores none delimiters. It solves the (1 + 2) * 3
problem by having magical token trees which can't be duplicated:
* https://rust-lang.zulipchat.com/#narrow/stream/185405-t-compiler.2Frust-analyzer/topic/TIL.20that.20token.20streams.20are.20magic
* https://rust-lang.zulipchat.com/#narrow/stream/131828-t-compiler/topic/Handling.20of.20Delimiter.3A.3ANone.20by.20the.20parser
_Second_, it's not like our implementation in rust-analyzer works. We
special-case expressions (as opposed to treating all kinds of $var
captures the same) and we don't know how parser error recovery should
work with these dollar-parenthesis.
So, in this PR we simplify the whole thing away by not pretending that
we are doing something proper and instead just explicitly special-casing
expressions by wrapping them into real `()`.
In the future, to maintain bug-parity with `rustc` what we are going to
do is probably adding an explicit `CAPTURED_EXPR` *token* which we can
explicitly account for in the parser.
If/when rustc starts handling delimiter=none properly, we'll port that
logic as well, in addition to special handling.
| -rw-r--r-- | crates/hir_def/src/macro_expansion_tests/mbe.rs | 93 | ||||
| -rw-r--r-- | crates/hir_def/src/macro_expansion_tests/mbe/regression.rs | 59 | ||||
| -rw-r--r-- | crates/mbe/src/expander.rs | 11 | ||||
| -rw-r--r-- | crates/mbe/src/expander/matcher.rs | 2 | ||||
| -rw-r--r-- | crates/mbe/src/expander/transcriber.rs | 11 | ||||
| -rw-r--r-- | crates/mbe/src/subtree_source.rs | 28 | ||||
| -rw-r--r-- | crates/mbe/src/syntax_bridge.rs | 91 |
7 files changed, 165 insertions, 130 deletions
diff --git a/crates/hir_def/src/macro_expansion_tests/mbe.rs b/crates/hir_def/src/macro_expansion_tests/mbe.rs index 7a866135b3..953750d360 100644 --- a/crates/hir_def/src/macro_expansion_tests/mbe.rs +++ b/crates/hir_def/src/macro_expansion_tests/mbe.rs @@ -317,32 +317,35 @@ macro_rules! m { ($ i:expr) => { fn bar() { $ i * 3; } } } fn bar() { - 1+2*3; + (1+2)*3; } // [email protected] "fn" // [email protected] "bar" // [email protected] "(" // [email protected] ")" // [email protected] "{" -// [email protected] "1" -// [email protected] "+" -// [email protected] "2" -// [email protected] "*" -// [email protected] "3" -// [email protected] ";" -// [email protected] "}" +// [email protected] "(" +// [email protected] "1" +// [email protected] "+" +// [email protected] "2" +// [email protected] ")" +// [email protected] "*" +// [email protected] "3" +// [email protected] ";" +// [email protected] "}" "#]], ) @@ -722,7 +725,7 @@ macro_rules! m { } fn bar() { - 2+2*baz(3).quux(); + (2+2*baz(3).quux()); } "#]], ) @@ -1370,42 +1373,48 @@ macro_rules! m { } /* parse error: expected identifier */ /* parse error: expected SEMICOLON */ +/* parse error: expected SEMICOLON */ +/* parse error: expected expression */ fn f() { - K::C("0"); + K::(C("0")); } // [email protected] "fn" // [email protected] "f" // [email protected] "(" // [email protected] ")" // [email protected] "{" // [email protected] "K" // [email protected] "::" -// [email protected] "C" -// [email protected] "(" -// [email protected] "\"0\"" -// [email protected] ")" -// [email protected] ";" -// [email protected] "}" +// [email protected] "(" +// [email protected] "C" +// [email protected] "(" +// [email protected] "\"0\"" +// [email protected] ")" +// [email protected] ")" +// [email protected] ";" +// [email protected] "}" "#]], ); @@ -1441,7 +1450,7 @@ fn f() { expect![[r#" macro_rules! m { ($expr:expr) => { map($expr) } } fn f() { - let _ = map(x+foo); + let _ = map((x+foo)); } "#]], ) diff --git a/crates/hir_def/src/macro_expansion_tests/mbe/regression.rs b/crates/hir_def/src/macro_expansion_tests/mbe/regression.rs index 563fe50588..2dff4adf2e 100644 --- a/crates/hir_def/src/macro_expansion_tests/mbe/regression.rs +++ b/crates/hir_def/src/macro_expansion_tests/mbe/regression.rs @@ -825,15 +825,18 @@ macro_rules! rgb_color { }; } /* parse error: expected type */ +/* parse error: expected R_PAREN */ /* parse error: expected R_ANGLE */ /* parse error: expected COMMA */ /* parse error: expected R_ANGLE */ /* parse error: expected SEMICOLON */ +/* parse error: expected SEMICOLON */ +/* parse error: expected expression */ pub fn new() { - let _ = 0as u32<<8+8; + let _ = 0as u32<<(8+8); } // [email protected] "pub" // [email protected] "fn" @@ -842,39 +845,45 @@ pub fn new() { // [email protected] "(" // [email protected] ")" // [email protected] "{" // [email protected] "let" // [email protected] "_" // [email protected] "=" // [email protected] "0" // [email protected] "as" // [email protected] "u32" // [email protected] "<" -// [email protected] "<" -// [email protected] "8" -// [email protected] "+" -// [email protected] "8" -// [email protected] ";" -// [email protected] "}" +// [email protected] "<" +// [email protected] "(" +// [email protected] "8" +// [email protected] "+" +// [email protected] "8" +// [email protected] ")" +// [email protected] ";" +// [email protected] "}" "#]], ); diff --git a/crates/mbe/src/expander.rs b/crates/mbe/src/expander.rs index 7244d21161..1e1bfa5505 100644 --- a/crates/mbe/src/expander.rs +++ b/crates/mbe/src/expander.rs @@ -110,7 +110,12 @@ enum Binding { enum Fragment { /// token fragments are just copy-pasted into the output Tokens(tt::TokenTree), - /// Ast fragments are inserted with fake delimiters, so as to make things - /// like `$i * 2` where `$i = 1 + 1` work as expectd. - Ast(tt::TokenTree), + /// Expr ast fragments are surrounded with `()` on insertion to preserve + /// precedence. Note that this impl is different from the one currently in + /// `rustc` -- `rustc` doesn't translate fragments into token trees at all. + /// + /// At one point in time, we tried to to use "fake" delimiters here a-la + /// proc-macro delimiter=none. As we later discovered, "none" delimiters are + /// tricky to handle in the parser, and rustc doesn't handle those either. + Expr(tt::TokenTree), } diff --git a/crates/mbe/src/expander/matcher.rs b/crates/mbe/src/expander/matcher.rs index 06c21dfd70..b2d3f038f5 100644 --- a/crates/mbe/src/expander/matcher.rs +++ b/crates/mbe/src/expander/matcher.rs @@ -736,7 +736,7 @@ fn match_meta_var(kind: &str, input: &mut TtIter) -> ExpandResult<Option<Fragmen } }; let result = input.expect_fragment(fragment); - result.map(|tt| if kind == "expr" { tt.map(Fragment::Ast) } else { tt.map(Fragment::Tokens) }) + result.map(|tt| if kind == "expr" { tt.map(Fragment::Expr) } else { tt.map(Fragment::Tokens) }) } fn collect_vars(buf: &mut Vec<SmolStr>, pattern: &MetaTemplate) { diff --git a/crates/mbe/src/expander/transcriber.rs b/crates/mbe/src/expander/transcriber.rs index 8140f25b06..0db17fea6e 100644 --- a/crates/mbe/src/expander/transcriber.rs +++ b/crates/mbe/src/expander/transcriber.rs @@ -238,7 +238,16 @@ fn expand_repeat( fn push_fragment(buf: &mut Vec<tt::TokenTree>, fragment: Fragment) { match fragment { Fragment::Tokens(tt::TokenTree::Subtree(tt)) => push_subtree(buf, tt), - Fragment::Tokens(tt) | Fragment::Ast(tt) => buf.push(tt), + Fragment::Expr(tt::TokenTree::Subtree(mut tt)) => { + if tt.delimiter.is_none() { + tt.delimiter = Some(tt::Delimiter { + id: tt::TokenId::unspecified(), + kind: tt::DelimiterKind::Parenthesis, + }) + } + buf.push(tt.into()) + } + Fragment::Tokens(tt) | Fragment::Expr(tt) => buf.push(tt), } } diff --git a/crates/mbe/src/subtree_source.rs b/crates/mbe/src/subtree_source.rs index 247616aa1b..6bdd787e30 100644 --- a/crates/mbe/src/subtree_source.rs +++ b/crates/mbe/src/subtree_source.rs @@ -52,17 +52,20 @@ impl<'a> SubtreeTokenSource { cursor.bump() } Some(tt::buffer::TokenTreeRef::Subtree(subtree, _)) => { - cached.push(convert_delim(subtree.delimiter_kind(), false)); + if let Some(d) = subtree.delimiter_kind() { + cached.push(convert_delim(d, false)); + } cursor.subtree().unwrap() } - None => { - if let Some(subtree) = cursor.end() { - cached.push(convert_delim(subtree.delimiter_kind(), true)); + None => match cursor.end() { + Some(subtree) => { + if let Some(d) = subtree.delimiter_kind() { + cached.push(convert_delim(d, true)); + } cursor.bump() - } else { - continue; } - } + None => continue, + }, }; } @@ -109,17 +112,16 @@ impl<'a> TokenSource for SubtreeTokenSource { } } -fn convert_delim(d: Option<tt::DelimiterKind>, closing: bool) -> TtToken { +fn convert_delim(d: tt::DelimiterKind, closing: bool) -> TtToken { let (kinds, texts) = match d { - Some(tt::DelimiterKind::Parenthesis) => ([T!['('], T![')']], "()"), - Some(tt::DelimiterKind::Brace) => ([T!['{'], T!['}']], "{}"), - Some(tt::DelimiterKind::Bracket) => ([T!['['], T![']']], "[]"), - None => ([L_DOLLAR, R_DOLLAR], ""), + tt::DelimiterKind::Parenthesis => ([T!['('], T![')']], "()"), + tt::DelimiterKind::Brace => ([T!['{'], T!['}']], "{}"), + tt::DelimiterKind::Bracket => ([T!['['], T![']']], "[]"), }; let idx = closing as usize; let kind = kinds[idx]; - let text = if !texts.is_empty() { &texts[idx..texts.len() - (1 - idx)] } else { "" }; + let text = &texts[idx..texts.len() - (1 - idx)]; TtToken { tt: Token { kind, is_jointed_to_next: false }, text: SmolStr::new(text) } } diff --git a/crates/mbe/src/syntax_bridge.rs b/crates/mbe/src/syntax_bridge.rs index 6bd57bb568..0b65fa171f 100644 --- a/crates/mbe/src/syntax_bridge.rs +++ b/crates/mbe/src/syntax_bridge.rs @@ -632,12 +632,11 @@ impl<'a> TtTreeSink<'a> { } } -fn delim_to_str(d: Option<tt::DelimiterKind>, closing: bool) -> &'static str { +fn delim_to_str(d: tt::DelimiterKind, closing: bool) -> &'static str { let texts = match d { - Some(tt::DelimiterKind::Parenthesis) => "()", - Some(tt::DelimiterKind::Brace) => "{}", - Some(tt::DelimiterKind::Bracket) => "[]", - None => return "", + tt::DelimiterKind::Parenthesis => "()", + tt::DelimiterKind::Brace => "{}", + tt::DelimiterKind::Bracket => "[]", }; let idx = closing as usize; @@ -646,10 +645,6 @@ fn delim_to_str(d: Option<tt::DelimiterKind>, closing: bool) -> &'static str { impl<'a> TreeSink for TtTreeSink<'a> { fn token(&mut self, kind: SyntaxKind, mut n_tokens: u8) { - if kind == L_DOLLAR || kind == R_DOLLAR { - self.cursor = self.cursor.bump_subtree(); - return; - } if kind == LIFETIME_IDENT { n_tokens = 2; } @@ -661,48 +656,54 @@ impl<'a> TreeSink for TtTreeSink<'a> { break; } last = self.cursor; - let text: &str = match self.cursor.token_tree() { - Some(tt::buffer::TokenTreeRef::Leaf(leaf, _)) => { - // Mark the range if needed - let (text, id) = match leaf { - tt::Leaf::Ident(ident) => (&ident.text, ident.id), - tt::Leaf::Punct(punct) => { - assert!(punct.char.is_ascii()); - let char = &(punct.char as u8); - tmp_str = SmolStr::new_inline( - std::str::from_utf8(std::slice::from_ref(char)).unwrap(), - ); - (&tmp_str, punct.id) + let text: &str = loop { + break match self.cursor.token_tree() { + Some(tt::buffer::TokenTreeRef::Leaf(leaf, _)) => { + // Mark the range if needed + let (text, id) = match leaf { + tt::Leaf::Ident(ident) => (&ident.text, ident.id), + tt::Leaf::Punct(punct) => { + assert!(punct.char.is_ascii()); + let char = &(punct.char as u8); + tmp_str = SmolStr::new_inline( + std::str::from_utf8(std::slice::from_ref(char)).unwrap(), + ); + (&tmp_str, punct.id) + } + tt::Leaf::Literal(lit) => (&lit.text, lit.id), + }; + let range = TextRange::at(self.text_pos, TextSize::of(text.as_str())); + self.token_map.insert(id, range); + self.cursor = self.cursor.bump(); + text + } + Some(tt::buffer::TokenTreeRef::Subtree(subtree, _)) => { + self.cursor = self.cursor.subtree().unwrap(); + match subtree.delimiter { + Some(d) => { + self.open_delims.insert(d.id, self.text_pos); + delim_to_str(d.kind, false) + } + None => continue, } - tt::Leaf::Literal(lit) => (&lit.text, lit.id), - }; - let range = TextRange::at(self.text_pos, TextSize::of(text.as_str())); - self.token_map.insert(id, range); - self.cursor = self.cursor.bump(); - text - } - Some(tt::buffer::TokenTreeRef::Subtree(subtree, _)) => { - self.cursor = self.cursor.subtree().unwrap(); - if let Some(id) = subtree.delimiter.map(|it| it.id) { - self.open_delims.insert(id, self.text_pos); } - delim_to_str(subtree.delimiter_kind(), false) - } - None => { - if let Some(parent) = self.cursor.end() { + None => { + let parent = self.cursor.end().unwrap(); self.cursor = self.cursor.bump(); - if let Some(id) = parent.delimiter.map(|it| it.id) { - if let Some(open_delim) = self.open_delims.get(&id) { - let open_range = TextRange::at(*open_delim, TextSize::of('(')); - let close_range = TextRange::at(self.text_pos, TextSize::of('(')); - self.token_map.insert_delim(id, open_range, close_range); + match parent.delimiter { + Some(d) => { + if let Some(open_delim) = self.open_delims.get(&d.id) { + let open_range = TextRange::at(*open_delim, TextSize::of('(')); + let close_range = + TextRange::at(self.text_pos, TextSize::of('(')); + self.token_map.insert_delim(d.id, open_range, close_range); + } + delim_to_str(d.kind, true) } + None => continue, } - delim_to_str(parent.delimiter_kind(), true) - } else { - continue; } - } + }; }; self.buf += text; self.text_pos += TextSize::of(text); |