Unnamed repository; edit this file 'description' to name the repository.
| -rw-r--r-- | crates/hir_def/src/macro_expansion_tests/mbe.rs | 93 | ||||
| -rw-r--r-- | crates/hir_def/src/macro_expansion_tests/mbe/regression.rs | 59 | ||||
| -rw-r--r-- | crates/mbe/src/expander.rs | 11 | ||||
| -rw-r--r-- | crates/mbe/src/expander/matcher.rs | 2 | ||||
| -rw-r--r-- | crates/mbe/src/expander/transcriber.rs | 11 | ||||
| -rw-r--r-- | crates/mbe/src/subtree_source.rs | 28 | ||||
| -rw-r--r-- | crates/mbe/src/syntax_bridge.rs | 91 | ||||
| -rw-r--r-- | crates/parser/src/grammar/expressions/atom.rs | 28 | ||||
| -rw-r--r-- | crates/parser/src/grammar/types.rs | 1 | ||||
| -rw-r--r-- | crates/parser/src/parser.rs | 16 | ||||
| -rw-r--r-- | crates/parser/src/syntax_kind/generated.rs | 2 | ||||
| -rw-r--r-- | crates/syntax/src/tests/ast_src.rs | 11 |
12 files changed, 170 insertions, 183 deletions
diff --git a/crates/hir_def/src/macro_expansion_tests/mbe.rs b/crates/hir_def/src/macro_expansion_tests/mbe.rs index 7a866135b3..953750d360 100644 --- a/crates/hir_def/src/macro_expansion_tests/mbe.rs +++ b/crates/hir_def/src/macro_expansion_tests/mbe.rs @@ -317,32 +317,35 @@ macro_rules! m { ($ i:expr) => { fn bar() { $ i * 3; } } } fn bar() { - 1+2*3; + (1+2)*3; } // [email protected] "fn" // [email protected] "bar" // [email protected] "(" // [email protected] ")" // [email protected] "{" -// [email protected] "1" -// [email protected] "+" -// [email protected] "2" -// [email protected] "*" -// [email protected] "3" -// [email protected] ";" -// [email protected] "}" +// [email protected] "(" +// [email protected] "1" +// [email protected] "+" +// [email protected] "2" +// [email protected] ")" +// [email protected] "*" +// [email protected] "3" +// [email protected] ";" +// [email protected] "}" "#]], ) @@ -722,7 +725,7 @@ macro_rules! m { } fn bar() { - 2+2*baz(3).quux(); + (2+2*baz(3).quux()); } "#]], ) @@ -1370,42 +1373,48 @@ macro_rules! m { } /* parse error: expected identifier */ /* parse error: expected SEMICOLON */ +/* parse error: expected SEMICOLON */ +/* parse error: expected expression */ fn f() { - K::C("0"); + K::(C("0")); } // [email protected] "fn" // [email protected] "f" // [email protected] "(" // [email protected] ")" // [email protected] "{" // [email protected] "K" // [email protected] "::" -// [email protected] "C" -// [email protected] "(" -// [email protected] "\"0\"" -// [email protected] ")" -// [email protected] ";" -// [email protected] "}" +// [email protected] "(" +// [email protected] "C" +// [email protected] "(" +// [email protected] "\"0\"" +// [email protected] ")" +// [email protected] ")" +// [email protected] ";" +// [email protected] "}" "#]], ); @@ -1441,7 +1450,7 @@ fn f() { expect![[r#" macro_rules! m { ($expr:expr) => { map($expr) } } fn f() { - let _ = map(x+foo); + let _ = map((x+foo)); } "#]], ) diff --git a/crates/hir_def/src/macro_expansion_tests/mbe/regression.rs b/crates/hir_def/src/macro_expansion_tests/mbe/regression.rs index 563fe50588..2dff4adf2e 100644 --- a/crates/hir_def/src/macro_expansion_tests/mbe/regression.rs +++ b/crates/hir_def/src/macro_expansion_tests/mbe/regression.rs @@ -825,15 +825,18 @@ macro_rules! rgb_color { }; } /* parse error: expected type */ +/* parse error: expected R_PAREN */ /* parse error: expected R_ANGLE */ /* parse error: expected COMMA */ /* parse error: expected R_ANGLE */ /* parse error: expected SEMICOLON */ +/* parse error: expected SEMICOLON */ +/* parse error: expected expression */ pub fn new() { - let _ = 0as u32<<8+8; + let _ = 0as u32<<(8+8); } // [email protected] "pub" // [email protected] "fn" @@ -842,39 +845,45 @@ pub fn new() { // [email protected] "(" // [email protected] ")" // [email protected] "{" // [email protected] "let" // [email protected] "_" // [email protected] "=" // [email protected] "0" // [email protected] "as" // [email protected] "u32" // [email protected] "<" -// [email protected] "<" -// [email protected] "8" -// [email protected] "+" -// [email protected] "8" -// [email protected] ";" -// [email protected] "}" +// [email protected] "<" +// [email protected] "(" +// [email protected] "8" +// [email protected] "+" +// [email protected] "8" +// [email protected] ")" +// [email protected] ";" +// [email protected] "}" "#]], ); diff --git a/crates/mbe/src/expander.rs b/crates/mbe/src/expander.rs index 7244d21161..1e1bfa5505 100644 --- a/crates/mbe/src/expander.rs +++ b/crates/mbe/src/expander.rs @@ -110,7 +110,12 @@ enum Binding { enum Fragment { /// token fragments are just copy-pasted into the output Tokens(tt::TokenTree), - /// Ast fragments are inserted with fake delimiters, so as to make things - /// like `$i * 2` where `$i = 1 + 1` work as expectd. - Ast(tt::TokenTree), + /// Expr ast fragments are surrounded with `()` on insertion to preserve + /// precedence. Note that this impl is different from the one currently in + /// `rustc` -- `rustc` doesn't translate fragments into token trees at all. + /// + /// At one point in time, we tried to to use "fake" delimiters here a-la + /// proc-macro delimiter=none. As we later discovered, "none" delimiters are + /// tricky to handle in the parser, and rustc doesn't handle those either. + Expr(tt::TokenTree), } diff --git a/crates/mbe/src/expander/matcher.rs b/crates/mbe/src/expander/matcher.rs index 06c21dfd70..b2d3f038f5 100644 --- a/crates/mbe/src/expander/matcher.rs +++ b/crates/mbe/src/expander/matcher.rs @@ -736,7 +736,7 @@ fn match_meta_var(kind: &str, input: &mut TtIter) -> ExpandResult<Option<Fragmen } }; let result = input.expect_fragment(fragment); - result.map(|tt| if kind == "expr" { tt.map(Fragment::Ast) } else { tt.map(Fragment::Tokens) }) + result.map(|tt| if kind == "expr" { tt.map(Fragment::Expr) } else { tt.map(Fragment::Tokens) }) } fn collect_vars(buf: &mut Vec<SmolStr>, pattern: &MetaTemplate) { diff --git a/crates/mbe/src/expander/transcriber.rs b/crates/mbe/src/expander/transcriber.rs index 8140f25b06..0db17fea6e 100644 --- a/crates/mbe/src/expander/transcriber.rs +++ b/crates/mbe/src/expander/transcriber.rs @@ -238,7 +238,16 @@ fn expand_repeat( fn push_fragment(buf: &mut Vec<tt::TokenTree>, fragment: Fragment) { match fragment { Fragment::Tokens(tt::TokenTree::Subtree(tt)) => push_subtree(buf, tt), - Fragment::Tokens(tt) | Fragment::Ast(tt) => buf.push(tt), + Fragment::Expr(tt::TokenTree::Subtree(mut tt)) => { + if tt.delimiter.is_none() { + tt.delimiter = Some(tt::Delimiter { + id: tt::TokenId::unspecified(), + kind: tt::DelimiterKind::Parenthesis, + }) + } + buf.push(tt.into()) + } + Fragment::Tokens(tt) | Fragment::Expr(tt) => buf.push(tt), } } diff --git a/crates/mbe/src/subtree_source.rs b/crates/mbe/src/subtree_source.rs index 247616aa1b..6bdd787e30 100644 --- a/crates/mbe/src/subtree_source.rs +++ b/crates/mbe/src/subtree_source.rs @@ -52,17 +52,20 @@ impl<'a> SubtreeTokenSource { cursor.bump() } Some(tt::buffer::TokenTreeRef::Subtree(subtree, _)) => { - cached.push(convert_delim(subtree.delimiter_kind(), false)); + if let Some(d) = subtree.delimiter_kind() { + cached.push(convert_delim(d, false)); + } cursor.subtree().unwrap() } - None => { - if let Some(subtree) = cursor.end() { - cached.push(convert_delim(subtree.delimiter_kind(), true)); + None => match cursor.end() { + Some(subtree) => { + if let Some(d) = subtree.delimiter_kind() { + cached.push(convert_delim(d, true)); + } cursor.bump() - } else { - continue; } - } + None => continue, + }, }; } @@ -109,17 +112,16 @@ impl<'a> TokenSource for SubtreeTokenSource { } } -fn convert_delim(d: Option<tt::DelimiterKind>, closing: bool) -> TtToken { +fn convert_delim(d: tt::DelimiterKind, closing: bool) -> TtToken { let (kinds, texts) = match d { - Some(tt::DelimiterKind::Parenthesis) => ([T!['('], T![')']], "()"), - Some(tt::DelimiterKind::Brace) => ([T!['{'], T!['}']], "{}"), - Some(tt::DelimiterKind::Bracket) => ([T!['['], T![']']], "[]"), - None => ([L_DOLLAR, R_DOLLAR], ""), + tt::DelimiterKind::Parenthesis => ([T!['('], T![')']], "()"), + tt::DelimiterKind::Brace => ([T!['{'], T!['}']], "{}"), + tt::DelimiterKind::Bracket => ([T!['['], T![']']], "[]"), }; let idx = closing as usize; let kind = kinds[idx]; - let text = if !texts.is_empty() { &texts[idx..texts.len() - (1 - idx)] } else { "" }; + let text = &texts[idx..texts.len() - (1 - idx)]; TtToken { tt: Token { kind, is_jointed_to_next: false }, text: SmolStr::new(text) } } diff --git a/crates/mbe/src/syntax_bridge.rs b/crates/mbe/src/syntax_bridge.rs index 6bd57bb568..0b65fa171f 100644 --- a/crates/mbe/src/syntax_bridge.rs +++ b/crates/mbe/src/syntax_bridge.rs @@ -632,12 +632,11 @@ impl<'a> TtTreeSink<'a> { } } -fn delim_to_str(d: Option<tt::DelimiterKind>, closing: bool) -> &'static str { +fn delim_to_str(d: tt::DelimiterKind, closing: bool) -> &'static str { let texts = match d { - Some(tt::DelimiterKind::Parenthesis) => "()", - Some(tt::DelimiterKind::Brace) => "{}", - Some(tt::DelimiterKind::Bracket) => "[]", - None => return "", + tt::DelimiterKind::Parenthesis => "()", + tt::DelimiterKind::Brace => "{}", + tt::DelimiterKind::Bracket => "[]", }; let idx = closing as usize; @@ -646,10 +645,6 @@ fn delim_to_str(d: Option<tt::DelimiterKind>, closing: bool) -> &'static str { impl<'a> TreeSink for TtTreeSink<'a> { fn token(&mut self, kind: SyntaxKind, mut n_tokens: u8) { - if kind == L_DOLLAR || kind == R_DOLLAR { - self.cursor = self.cursor.bump_subtree(); - return; - } if kind == LIFETIME_IDENT { n_tokens = 2; } @@ -661,48 +656,54 @@ impl<'a> TreeSink for TtTreeSink<'a> { break; } last = self.cursor; - let text: &str = match self.cursor.token_tree() { - Some(tt::buffer::TokenTreeRef::Leaf(leaf, _)) => { - // Mark the range if needed - let (text, id) = match leaf { - tt::Leaf::Ident(ident) => (&ident.text, ident.id), - tt::Leaf::Punct(punct) => { - assert!(punct.char.is_ascii()); - let char = &(punct.char as u8); - tmp_str = SmolStr::new_inline( - std::str::from_utf8(std::slice::from_ref(char)).unwrap(), - ); - (&tmp_str, punct.id) + let text: &str = loop { + break match self.cursor.token_tree() { + Some(tt::buffer::TokenTreeRef::Leaf(leaf, _)) => { + // Mark the range if needed + let (text, id) = match leaf { + tt::Leaf::Ident(ident) => (&ident.text, ident.id), + tt::Leaf::Punct(punct) => { + assert!(punct.char.is_ascii()); + let char = &(punct.char as u8); + tmp_str = SmolStr::new_inline( + std::str::from_utf8(std::slice::from_ref(char)).unwrap(), + ); + (&tmp_str, punct.id) + } + tt::Leaf::Literal(lit) => (&lit.text, lit.id), + }; + let range = TextRange::at(self.text_pos, TextSize::of(text.as_str())); + self.token_map.insert(id, range); + self.cursor = self.cursor.bump(); + text + } + Some(tt::buffer::TokenTreeRef::Subtree(subtree, _)) => { + self.cursor = self.cursor.subtree().unwrap(); + match subtree.delimiter { + Some(d) => { + self.open_delims.insert(d.id, self.text_pos); + delim_to_str(d.kind, false) + } + None => continue, } - tt::Leaf::Literal(lit) => (&lit.text, lit.id), - }; - let range = TextRange::at(self.text_pos, TextSize::of(text.as_str())); - self.token_map.insert(id, range); - self.cursor = self.cursor.bump(); - text - } - Some(tt::buffer::TokenTreeRef::Subtree(subtree, _)) => { - self.cursor = self.cursor.subtree().unwrap(); - if let Some(id) = subtree.delimiter.map(|it| it.id) { - self.open_delims.insert(id, self.text_pos); } - delim_to_str(subtree.delimiter_kind(), false) - } - None => { - if let Some(parent) = self.cursor.end() { + None => { + let parent = self.cursor.end().unwrap(); self.cursor = self.cursor.bump(); - if let Some(id) = parent.delimiter.map(|it| it.id) { - if let Some(open_delim) = self.open_delims.get(&id) { - let open_range = TextRange::at(*open_delim, TextSize::of('(')); - let close_range = TextRange::at(self.text_pos, TextSize::of('(')); - self.token_map.insert_delim(id, open_range, close_range); + match parent.delimiter { + Some(d) => { + if let Some(open_delim) = self.open_delims.get(&d.id) { + let open_range = TextRange::at(*open_delim, TextSize::of('(')); + let close_range = + TextRange::at(self.text_pos, TextSize::of('(')); + self.token_map.insert_delim(d.id, open_range, close_range); + } + delim_to_str(d.kind, true) } + None => continue, } - delim_to_str(parent.delimiter_kind(), true) - } else { - continue; } - } + }; }; self.buf += text; self.text_pos += TextSize::of(text); diff --git a/crates/parser/src/grammar/expressions/atom.rs b/crates/parser/src/grammar/expressions/atom.rs index 7075ae297f..640caa0778 100644 --- a/crates/parser/src/grammar/expressions/atom.rs +++ b/crates/parser/src/grammar/expressions/atom.rs @@ -39,7 +39,6 @@ pub(super) const ATOM_EXPR_FIRST: TokenSet = T!['('], T!['{'], T!['['], - L_DOLLAR, T![|], T![move], T![box], @@ -59,7 +58,7 @@ pub(super) const ATOM_EXPR_FIRST: TokenSet = LIFETIME_IDENT, ])); -const EXPR_RECOVERY_SET: TokenSet = TokenSet::new(&[T![let], R_DOLLAR]); +const EXPR_RECOVERY_SET: TokenSet = TokenSet::new(&[T![let]]); pub(super) fn atom_expr(p: &mut Parser, r: Restrictions) -> Option<(CompletedMarker, BlockLike)> { if let Some(m) = literal(p) { @@ -72,7 +71,6 @@ pub(super) fn atom_expr(p: &mut Parser, r: Restrictions) -> Option<(CompletedMar let done = match p.current() { T!['('] => tuple_expr(p), T!['['] => array_expr(p), - L_DOLLAR => meta_var_expr(p), T![|] => closure_expr(p), T![move] if la == T![|] => closure_expr(p), T![async] if la == T![|] || (la == T![move] && p.nth(2) == T![|]) => closure_expr(p), @@ -622,27 +620,3 @@ fn box_expr(p: &mut Parser, m: Option<Marker>) -> CompletedMarker { } m.complete(p, BOX_EXPR) } - -/// Expression from `$var` macro expansion, wrapped in dollars -fn meta_var_expr(p: &mut Parser) -> CompletedMarker { - assert!(p.at(L_DOLLAR)); - let m = p.start(); - p.bump(L_DOLLAR); - let expr = expr_bp(p, None, Restrictions { forbid_structs: false, prefer_stmt: false }, 1); - - match (expr, p.current()) { - (Some((cm, _)), R_DOLLAR) => { - p.bump(R_DOLLAR); - // FIXME: this leaves the dollar hanging in the air... - m.abandon(p); - cm - } - _ => { - while !p.at(R_DOLLAR) { - p.bump_any(); - } - p.bump(R_DOLLAR); - m.complete(p, ERROR) - } - } -} diff --git a/crates/parser/src/grammar/types.rs b/crates/parser/src/grammar/types.rs index 1a6fb651cb..05c1aea5c4 100644 --- a/crates/parser/src/grammar/types.rs +++ b/crates/parser/src/grammar/types.rs @@ -19,7 +19,6 @@ pub(super) const TYPE_FIRST: TokenSet = paths::PATH_FIRST.union(TokenSet::new(&[ const TYPE_RECOVERY_SET: TokenSet = TokenSet::new(&[ T![')'], T![,], - L_DOLLAR, // test_err struct_field_recover // struct S { f pub g: () } T![pub], diff --git a/crates/parser/src/parser.rs b/crates/parser/src/parser.rs index 89dfea52f8..5820ffd77d 100644 --- a/crates/parser/src/parser.rs +++ b/crates/parser/src/parser.rs @@ -7,7 +7,7 @@ use drop_bomb::DropBomb; use crate::{ event::Event, ParseError, - SyntaxKind::{self, EOF, ERROR, L_DOLLAR, R_DOLLAR, TOMBSTONE}, + SyntaxKind::{self, EOF, ERROR, TOMBSTONE}, TokenSet, TokenSource, T, }; @@ -215,23 +215,13 @@ impl<'t> Parser<'t> { /// Create an error node and consume the next token. pub(crate) fn err_and_bump(&mut self, message: &str) { - match self.current() { - L_DOLLAR | R_DOLLAR => { - let m = self.start(); - self.error(message); - self.bump_any(); - m.complete(self, ERROR); - } - _ => { - self.err_recover(message, TokenSet::EMPTY); - } - } + self.err_recover(message, TokenSet::EMPTY); } /// Create an error node and consume the next token. pub(crate) fn err_recover(&mut self, message: &str, recovery: TokenSet) { match self.current() { - T!['{'] | T!['}'] | L_DOLLAR | R_DOLLAR => { + T!['{'] | T!['}'] => { self.error(message); return; } diff --git a/crates/parser/src/syntax_kind/generated.rs b/crates/parser/src/syntax_kind/generated.rs index a9010c0c31..99e7651906 100644 --- a/crates/parser/src/syntax_kind/generated.rs +++ b/crates/parser/src/syntax_kind/generated.rs @@ -120,8 +120,6 @@ pub enum SyntaxKind { LIFETIME_IDENT, COMMENT, SHEBANG, - L_DOLLAR, - R_DOLLAR, SOURCE_FILE, STRUCT, UNION, diff --git a/crates/syntax/src/tests/ast_src.rs b/crates/syntax/src/tests/ast_src.rs index 2344fc59d1..c0f1d5ef56 100644 --- a/crates/syntax/src/tests/ast_src.rs +++ b/crates/syntax/src/tests/ast_src.rs @@ -72,16 +72,7 @@ pub(crate) const KINDS_SRC: KindsSrc = KindsSrc { ], contextual_keywords: &["auto", "default", "existential", "union", "raw", "macro_rules"], literals: &["INT_NUMBER", "FLOAT_NUMBER", "CHAR", "BYTE", "STRING", "BYTE_STRING"], - tokens: &[ - "ERROR", - "IDENT", - "WHITESPACE", - "LIFETIME_IDENT", - "COMMENT", - "SHEBANG", - "L_DOLLAR", - "R_DOLLAR", - ], + tokens: &["ERROR", "IDENT", "WHITESPACE", "LIFETIME_IDENT", "COMMENT", "SHEBANG"], nodes: &[ "SOURCE_FILE", "STRUCT", |