Unnamed repository; edit this file 'description' to name the repository.
Diffstat (limited to 'crates/mbe/src/syntax_bridge.rs')
| -rw-r--r-- | crates/mbe/src/syntax_bridge.rs | 89 |
1 files changed, 62 insertions, 27 deletions
diff --git a/crates/mbe/src/syntax_bridge.rs b/crates/mbe/src/syntax_bridge.rs index e4c56565b9..cf53c16726 100644 --- a/crates/mbe/src/syntax_bridge.rs +++ b/crates/mbe/src/syntax_bridge.rs @@ -12,6 +12,9 @@ use tt::buffer::{Cursor, TokenBuffer}; use crate::{to_parser_input::to_parser_input, tt_iter::TtIter, TokenMap}; +#[cfg(test)] +mod tests; + /// Convert the syntax node to a `TokenTree` (what macro /// will consume). pub fn syntax_node_to_token_tree(node: &SyntaxNode) -> (tt::Subtree, TokenMap) { @@ -35,7 +38,7 @@ pub fn syntax_node_to_token_tree_with_modifications( append: FxHashMap<SyntaxElement, Vec<SyntheticToken>>, ) -> (tt::Subtree, TokenMap, u32) { let global_offset = node.text_range().start(); - let mut c = Convertor::new(node, global_offset, existing_token_map, next_id, replace, append); + let mut c = Converter::new(node, global_offset, existing_token_map, next_id, replace, append); let subtree = convert_tokens(&mut c); c.id_alloc.map.shrink_to_fit(); always!(c.replace.is_empty(), "replace: {:?}", c.replace); @@ -100,7 +103,7 @@ pub fn parse_to_token_tree(text: &str) -> Option<(tt::Subtree, TokenMap)> { return None; } - let mut conv = RawConvertor { + let mut conv = RawConverter { lexed, pos: 0, id_alloc: TokenIdAlloc { @@ -148,7 +151,7 @@ pub fn parse_exprs_with_sep(tt: &tt::Subtree, sep: char) -> Vec<tt::Subtree> { res } -fn convert_tokens<C: TokenConvertor>(conv: &mut C) -> tt::Subtree { +fn convert_tokens<C: TokenConverter>(conv: &mut C) -> tt::Subtree { struct StackEntry { subtree: tt::Subtree, idx: usize, @@ -228,7 +231,7 @@ fn convert_tokens<C: TokenConvertor>(conv: &mut C) -> tt::Subtree { } let spacing = match conv.peek().map(|next| next.kind(conv)) { - Some(kind) if !kind.is_trivia() => tt::Spacing::Joint, + Some(kind) if is_single_token_op(kind) => tt::Spacing::Joint, _ => tt::Spacing::Alone, }; let char = match token.to_char(conv) { @@ -307,6 +310,35 @@ fn convert_tokens<C: TokenConvertor>(conv: &mut C) -> tt::Subtree { } } +fn is_single_token_op(kind: SyntaxKind) -> bool { + matches!( + kind, + EQ | L_ANGLE + | R_ANGLE + | BANG + | AMP + | PIPE + | TILDE + | AT + | DOT + | COMMA + | SEMICOLON + | COLON + | POUND + | DOLLAR + | QUESTION + | PLUS + | MINUS + | STAR + | SLASH + | PERCENT + | CARET + // LIFETIME_IDENT will be split into a sequence of `'` (a single quote) and an + // identifier. + | LIFETIME_IDENT + ) +} + /// Returns the textual content of a doc comment block as a quoted string /// That is, strips leading `///` (or `/**`, etc) /// and strips the ending `*/` @@ -425,8 +457,8 @@ impl TokenIdAlloc { } } -/// A raw token (straight from lexer) convertor -struct RawConvertor<'a> { +/// A raw token (straight from lexer) converter +struct RawConverter<'a> { lexed: parser::LexedStr<'a>, pos: usize, id_alloc: TokenIdAlloc, @@ -442,7 +474,7 @@ trait SrcToken<Ctx>: std::fmt::Debug { fn synthetic_id(&self, ctx: &Ctx) -> Option<SyntheticTokenId>; } -trait TokenConvertor: Sized { +trait TokenConverter: Sized { type Token: SrcToken<Self>; fn convert_doc_comment(&self, token: &Self::Token) -> Option<Vec<tt::TokenTree>>; @@ -454,25 +486,25 @@ trait TokenConvertor: Sized { fn id_alloc(&mut self) -> &mut TokenIdAlloc; } -impl<'a> SrcToken<RawConvertor<'a>> for usize { - fn kind(&self, ctx: &RawConvertor<'a>) -> SyntaxKind { +impl<'a> SrcToken<RawConverter<'a>> for usize { + fn kind(&self, ctx: &RawConverter<'a>) -> SyntaxKind { ctx.lexed.kind(*self) } - fn to_char(&self, ctx: &RawConvertor<'a>) -> Option<char> { + fn to_char(&self, ctx: &RawConverter<'a>) -> Option<char> { ctx.lexed.text(*self).chars().next() } - fn to_text(&self, ctx: &RawConvertor<'_>) -> SmolStr { + fn to_text(&self, ctx: &RawConverter<'_>) -> SmolStr { ctx.lexed.text(*self).into() } - fn synthetic_id(&self, _ctx: &RawConvertor<'a>) -> Option<SyntheticTokenId> { + fn synthetic_id(&self, _ctx: &RawConverter<'a>) -> Option<SyntheticTokenId> { None } } -impl<'a> TokenConvertor for RawConvertor<'a> { +impl<'a> TokenConverter for RawConverter<'a> { type Token = usize; fn convert_doc_comment(&self, &token: &usize) -> Option<Vec<tt::TokenTree>> { @@ -504,7 +536,7 @@ impl<'a> TokenConvertor for RawConvertor<'a> { } } -struct Convertor { +struct Converter { id_alloc: TokenIdAlloc, current: Option<SyntaxToken>, current_synthetic: Vec<SyntheticToken>, @@ -515,7 +547,7 @@ struct Convertor { punct_offset: Option<(SyntaxToken, TextSize)>, } -impl Convertor { +impl Converter { fn new( node: &SyntaxNode, global_offset: TextSize, @@ -523,11 +555,11 @@ impl Convertor { next_id: u32, mut replace: FxHashMap<SyntaxElement, Vec<SyntheticToken>>, mut append: FxHashMap<SyntaxElement, Vec<SyntheticToken>>, - ) -> Convertor { + ) -> Converter { let range = node.text_range(); let mut preorder = node.preorder_with_tokens(); let (first, synthetic) = Self::next_token(&mut preorder, &mut replace, &mut append); - Convertor { + Converter { id_alloc: { TokenIdAlloc { map: existing_token_map, global_offset, next_id } }, current: first, current_synthetic: synthetic, @@ -590,15 +622,15 @@ impl SynToken { } } -impl SrcToken<Convertor> for SynToken { - fn kind(&self, _ctx: &Convertor) -> SyntaxKind { +impl SrcToken<Converter> for SynToken { + fn kind(&self, ctx: &Converter) -> SyntaxKind { match self { SynToken::Ordinary(token) => token.kind(), - SynToken::Punch(token, _) => token.kind(), + SynToken::Punch(..) => SyntaxKind::from_char(self.to_char(ctx).unwrap()).unwrap(), SynToken::Synthetic(token) => token.kind, } } - fn to_char(&self, _ctx: &Convertor) -> Option<char> { + fn to_char(&self, _ctx: &Converter) -> Option<char> { match self { SynToken::Ordinary(_) => None, SynToken::Punch(it, i) => it.text().chars().nth((*i).into()), @@ -606,7 +638,7 @@ impl SrcToken<Convertor> for SynToken { SynToken::Synthetic(_) => None, } } - fn to_text(&self, _ctx: &Convertor) -> SmolStr { + fn to_text(&self, _ctx: &Converter) -> SmolStr { match self { SynToken::Ordinary(token) => token.text().into(), SynToken::Punch(token, _) => token.text().into(), @@ -614,7 +646,7 @@ impl SrcToken<Convertor> for SynToken { } } - fn synthetic_id(&self, _ctx: &Convertor) -> Option<SyntheticTokenId> { + fn synthetic_id(&self, _ctx: &Converter) -> Option<SyntheticTokenId> { match self { SynToken::Synthetic(token) => Some(token.id), _ => None, @@ -622,7 +654,7 @@ impl SrcToken<Convertor> for SynToken { } } -impl TokenConvertor for Convertor { +impl TokenConverter for Converter { type Token = SynToken; fn convert_doc_comment(&self, token: &Self::Token) -> Option<Vec<tt::TokenTree>> { convert_doc_comment(token.token()?) @@ -651,7 +683,7 @@ impl TokenConvertor for Convertor { } let curr = self.current.clone()?; - if !&self.range.contains_range(curr.text_range()) { + if !self.range.contains_range(curr.text_range()) { return None; } let (new_current, new_synth) = @@ -809,12 +841,15 @@ impl<'a> TtTreeSink<'a> { let next = last.bump(); if let ( Some(tt::buffer::TokenTreeRef::Leaf(tt::Leaf::Punct(curr), _)), - Some(tt::buffer::TokenTreeRef::Leaf(tt::Leaf::Punct(_), _)), + Some(tt::buffer::TokenTreeRef::Leaf(tt::Leaf::Punct(next), _)), ) = (last.token_tree(), next.token_tree()) { // Note: We always assume the semi-colon would be the last token in // other parts of RA such that we don't add whitespace here. - if curr.spacing == tt::Spacing::Alone && curr.char != ';' { + // + // When `next` is a `Punct` of `'`, that's a part of a lifetime identifier so we don't + // need to add whitespace either. + if curr.spacing == tt::Spacing::Alone && curr.char != ';' && next.char != '\'' { self.inner.token(WHITESPACE, " "); self.text_pos += TextSize::of(' '); } |