Unnamed repository; edit this file 'description' to name the repository.
Diffstat (limited to 'crates/mbe/src/expander/matcher.rs')
| -rw-r--r-- | crates/mbe/src/expander/matcher.rs | 227 |
1 files changed, 124 insertions, 103 deletions
diff --git a/crates/mbe/src/expander/matcher.rs b/crates/mbe/src/expander/matcher.rs index 3f656df25f..88eae136f7 100644 --- a/crates/mbe/src/expander/matcher.rs +++ b/crates/mbe/src/expander/matcher.rs @@ -68,7 +68,7 @@ use crate::{ expander::{Binding, Bindings, ExpandResult, Fragment}, parser::{MetaVarKind, Op, RepeatKind, Separator}, tt_iter::TtIter, - ExpandError, MetaTemplate, + ExpandError, MetaTemplate, ValueResult, }; impl Bindings { @@ -321,8 +321,8 @@ struct MatchState<'t> { /// The KleeneOp of this sequence if we are in a repetition. sep_kind: Option<RepeatKind>, - /// Number of tokens of separator parsed - sep_parsed: Option<usize>, + /// Whether we already matched separator token. + sep_matched: bool, /// Matched meta variables bindings bindings: BindingsIdx, @@ -387,7 +387,7 @@ fn match_loop_inner<'t>( None => { // We are at or past the end of the matcher of `item`. if let Some(up) = &item.up { - if item.sep_parsed.is_none() { + if !item.sep_matched { // Get the `up` matcher let mut new_pos = (**up).clone(); new_pos.bindings = bindings_builder.copy(&new_pos.bindings); @@ -401,14 +401,17 @@ fn match_loop_inner<'t>( } // Check if we need a separator. - // We check the separator one by one - let sep_idx = item.sep_parsed.unwrap_or(0); - let sep_len = item.sep.as_ref().map_or(0, Separator::tt_count); - if item.sep.is_some() && sep_idx != sep_len { + if item.sep.is_some() && !item.sep_matched { let sep = item.sep.as_ref().unwrap(); - if src.clone().expect_separator(sep, sep_idx) { + let mut fork = src.clone(); + if fork.expect_separator(sep) { + // HACK: here we use `meta_result` to pass `TtIter` back to caller because + // it might have been advanced multiple times. `ValueResult` is + // insignificant. + item.meta_result = Some((fork, ValueResult::ok(None))); item.dot.next(); - item.sep_parsed = Some(sep_idx + 1); + // item.sep_parsed = Some(sep_len); + item.sep_matched = true; try_push!(next_items, item); } } @@ -416,7 +419,7 @@ fn match_loop_inner<'t>( // and try to match again UNLESS we are only allowed to have _one_ repetition. else if item.sep_kind != Some(RepeatKind::ZeroOrOne) { item.dot = item.dot.reset(); - item.sep_parsed = None; + item.sep_matched = false; bindings_builder.push_default(&mut item.bindings); cur_items.push(item); } @@ -451,7 +454,7 @@ fn match_loop_inner<'t>( up: Some(Box::new(item)), sep: separator.clone(), sep_kind: Some(*kind), - sep_parsed: None, + sep_matched: false, bindings: bindings_builder.alloc(), meta_result: None, is_error: false, @@ -500,18 +503,69 @@ fn match_loop_inner<'t>( } } } - OpDelimited::Op(Op::Leaf(leaf)) => { - if let Err(err) = match_leaf(leaf, &mut src.clone()) { - res.add_err(err); + OpDelimited::Op(Op::Literal(lhs)) => { + if let Ok(rhs) = src.clone().expect_leaf() { + if matches!(rhs, tt::Leaf::Literal(it) if it.text == lhs.text) { + item.dot.next(); + } else { + res.add_err(ExpandError::UnexpectedToken); + item.is_error = true; + } + } else { + res.add_err(ExpandError::binding_error(format!("expected literal: `{lhs}`"))); item.is_error = true; + } + try_push!(next_items, item); + } + OpDelimited::Op(Op::Ident(lhs)) => { + if let Ok(rhs) = src.clone().expect_leaf() { + if matches!(rhs, tt::Leaf::Ident(it) if it.text == lhs.text) { + item.dot.next(); + } else { + res.add_err(ExpandError::UnexpectedToken); + item.is_error = true; + } } else { - item.dot.next(); + res.add_err(ExpandError::binding_error(format!("expected ident: `{lhs}`"))); + item.is_error = true; } try_push!(next_items, item); } + OpDelimited::Op(Op::Punct(lhs)) => { + let mut fork = src.clone(); + let error = if let Ok(rhs) = fork.expect_glued_punct() { + let first_is_single_quote = rhs[0].char == '\''; + let lhs = lhs.iter().map(|it| it.char); + let rhs = rhs.iter().map(|it| it.char); + if lhs.clone().eq(rhs) { + // HACK: here we use `meta_result` to pass `TtIter` back to caller because + // it might have been advanced multiple times. `ValueResult` is + // insignificant. + item.meta_result = Some((fork, ValueResult::ok(None))); + item.dot.next(); + next_items.push(item); + continue; + } + + if first_is_single_quote { + // If the first punct token is a single quote, that's a part of a lifetime + // ident, not a punct. + ExpandError::UnexpectedToken + } else { + let lhs: SmolStr = lhs.collect(); + ExpandError::binding_error(format!("expected punct: `{lhs}`")) + } + } else { + ExpandError::UnexpectedToken + }; + + res.add_err(error); + item.is_error = true; + error_items.push(item); + } OpDelimited::Op(Op::Ignore { .. } | Op::Index { .. }) => {} OpDelimited::Open => { - if matches!(src.clone().next(), Some(tt::TokenTree::Subtree(..))) { + if matches!(src.peek_n(0), Some(tt::TokenTree::Subtree(..))) { item.dot.next(); try_push!(next_items, item); } @@ -541,7 +595,7 @@ fn match_loop(pattern: &MetaTemplate, src: &tt::Subtree) -> Match { up: None, sep: None, sep_kind: None, - sep_parsed: None, + sep_matched: false, bindings: bindings_builder.alloc(), is_error: false, meta_result: None, @@ -616,21 +670,33 @@ fn match_loop(pattern: &MetaTemplate, src: &tt::Subtree) -> Match { } // Dump all possible `next_items` into `cur_items` for the next iteration. else if !next_items.is_empty() { - // Now process the next token - cur_items.extend(next_items.drain(..)); - - match src.next() { - Some(tt::TokenTree::Subtree(subtree)) => { - stack.push(src.clone()); - src = TtIter::new(subtree); + if let Some((iter, _)) = next_items[0].meta_result.take() { + // We've matched a possibly "glued" punct. The matched punct (hence + // `meta_result` also) must be the same for all items. + // FIXME: If there are multiple items, it's definitely redundant (and it's hacky! + // `meta_result` isn't supposed to be used this way). + + // We already bumped, so no need to call `.next()` like in the other branch. + src = iter; + for item in next_items.iter_mut() { + item.meta_result = None; } - None => { - if let Some(iter) = stack.pop() { - src = iter; + } else { + match src.next() { + Some(tt::TokenTree::Subtree(subtree)) => { + stack.push(src.clone()); + src = TtIter::new(subtree); } + None => { + if let Some(iter) = stack.pop() { + src = iter; + } + } + _ => (), } - _ => (), } + // Now process the next token + cur_items.extend(next_items.drain(..)); } // Finally, we have the case where we need to call the black-box parser to get some // nonterminal. @@ -663,27 +729,6 @@ fn match_loop(pattern: &MetaTemplate, src: &tt::Subtree) -> Match { } } -fn match_leaf(lhs: &tt::Leaf, src: &mut TtIter<'_>) -> Result<(), ExpandError> { - let rhs = src - .expect_leaf() - .map_err(|()| ExpandError::binding_error(format!("expected leaf: `{lhs}`")))?; - match (lhs, rhs) { - ( - tt::Leaf::Punct(tt::Punct { char: lhs, .. }), - tt::Leaf::Punct(tt::Punct { char: rhs, .. }), - ) if lhs == rhs => Ok(()), - ( - tt::Leaf::Ident(tt::Ident { text: lhs, .. }), - tt::Leaf::Ident(tt::Ident { text: rhs, .. }), - ) if lhs == rhs => Ok(()), - ( - tt::Leaf::Literal(tt::Literal { text: lhs, .. }), - tt::Leaf::Literal(tt::Literal { text: rhs, .. }), - ) if lhs == rhs => Ok(()), - _ => Err(ExpandError::UnexpectedToken), - } -} - fn match_meta_var(kind: MetaVarKind, input: &mut TtIter<'_>) -> ExpandResult<Option<Fragment>> { let fragment = match kind { MetaVarKind::Path => parser::PrefixEntryPoint::Path, @@ -698,12 +743,16 @@ fn match_meta_var(kind: MetaVarKind, input: &mut TtIter<'_>) -> ExpandResult<Opt MetaVarKind::Item => parser::PrefixEntryPoint::Item, MetaVarKind::Vis => parser::PrefixEntryPoint::Vis, MetaVarKind::Expr => { - // `expr` should not match underscores. + // `expr` should not match underscores, let expressions, or inline const. The latter + // two are for [backwards compatibility][0]. // HACK: Macro expansion should not be done using "rollback and try another alternative". - // rustc [explicitly checks the next token][0]. - // [0]: https://github.com/rust-lang/rust/blob/f0c4da499/compiler/rustc_expand/src/mbe/macro_parser.rs#L576 + // rustc [explicitly checks the next token][1]. + // [0]: https://github.com/rust-lang/rust/issues/86730 + // [1]: https://github.com/rust-lang/rust/blob/f0c4da499/compiler/rustc_expand/src/mbe/macro_parser.rs#L576 match input.peek_n(0) { - Some(tt::TokenTree::Leaf(tt::Leaf::Ident(it))) if it.text == "_" => { + Some(tt::TokenTree::Leaf(tt::Leaf::Ident(it))) + if it.text == "_" || it.text == "let" || it.text == "const" => + { return ExpandResult::only_err(ExpandError::NoMatchingRule) } _ => {} @@ -752,10 +801,10 @@ fn collect_vars(collector_fun: &mut impl FnMut(SmolStr), pattern: &MetaTemplate) for op in pattern.iter() { match op { Op::Var { name, .. } => collector_fun(name.clone()), - Op::Leaf(_) => (), Op::Subtree { tokens, .. } => collect_vars(collector_fun, tokens), Op::Repeat { tokens, .. } => collect_vars(collector_fun, tokens), - Op::Ignore { .. } | Op::Index { .. } => {} + Op::Ignore { .. } | Op::Index { .. } | Op::Literal(_) | Op::Ident(_) | Op::Punct(_) => { + } } } } @@ -818,14 +867,14 @@ impl<'a> Iterator for OpDelimitedIter<'a> { } impl<'a> TtIter<'a> { - fn expect_separator(&mut self, separator: &Separator, idx: usize) -> bool { + fn expect_separator(&mut self, separator: &Separator) -> bool { let mut fork = self.clone(); let ok = match separator { - Separator::Ident(lhs) if idx == 0 => match fork.expect_ident_or_underscore() { + Separator::Ident(lhs) => match fork.expect_ident_or_underscore() { Ok(rhs) => rhs.text == lhs.text, Err(_) => false, }, - Separator::Literal(lhs) if idx == 0 => match fork.expect_literal() { + Separator::Literal(lhs) => match fork.expect_literal() { Ok(rhs) => match rhs { tt::Leaf::Literal(rhs) => rhs.text == lhs.text, tt::Leaf::Ident(rhs) => rhs.text == lhs.text, @@ -833,11 +882,14 @@ impl<'a> TtIter<'a> { }, Err(_) => false, }, - Separator::Puncts(lhss) if idx < lhss.len() => match fork.expect_punct() { - Ok(rhs) => rhs.char == lhss[idx].char, + Separator::Puncts(lhs) => match fork.expect_glued_punct() { + Ok(rhs) => { + let lhs = lhs.iter().map(|it| it.char); + let rhs = rhs.iter().map(|it| it.char); + lhs.eq(rhs) + } Err(_) => false, }, - _ => false, }; if ok { *self = fork; @@ -846,52 +898,21 @@ impl<'a> TtIter<'a> { } fn expect_tt(&mut self) -> Result<tt::TokenTree, ()> { - match self.peek_n(0) { - Some(tt::TokenTree::Leaf(tt::Leaf::Punct(punct))) if punct.char == '\'' => { - return self.expect_lifetime(); - } - _ => (), - } - - let tt = self.next().ok_or(())?.clone(); - let punct = match tt { - tt::TokenTree::Leaf(tt::Leaf::Punct(punct)) if punct.spacing == tt::Spacing::Joint => { - punct - } - _ => return Ok(tt), - }; - - let (second, third) = match (self.peek_n(0), self.peek_n(1)) { - ( - Some(tt::TokenTree::Leaf(tt::Leaf::Punct(p2))), - Some(tt::TokenTree::Leaf(tt::Leaf::Punct(p3))), - ) if p2.spacing == tt::Spacing::Joint => (p2.char, Some(p3.char)), - (Some(tt::TokenTree::Leaf(tt::Leaf::Punct(p2))), _) => (p2.char, None), - _ => return Ok(tt), - }; - - match (punct.char, second, third) { - ('.', '.', Some('.' | '=')) | ('<', '<', Some('=')) | ('>', '>', Some('=')) => { - let tt2 = self.next().unwrap().clone(); - let tt3 = self.next().unwrap().clone(); - Ok(tt::Subtree { delimiter: None, token_trees: vec![tt, tt2, tt3] }.into()) - } - ('-' | '!' | '*' | '/' | '&' | '%' | '^' | '+' | '<' | '=' | '>' | '|', '=', _) - | ('-' | '=' | '>', '>', _) - | (':', ':', _) - | ('.', '.', _) - | ('&', '&', _) - | ('<', '<', _) - | ('|', '|', _) => { - let tt2 = self.next().unwrap().clone(); - Ok(tt::Subtree { delimiter: None, token_trees: vec![tt, tt2] }.into()) + if let Some(tt::TokenTree::Leaf(tt::Leaf::Punct(punct))) = self.peek_n(0) { + if punct.char == '\'' { + self.expect_lifetime() + } else { + let puncts = self.expect_glued_punct()?; + let token_trees = puncts.into_iter().map(|p| tt::Leaf::Punct(p).into()).collect(); + Ok(tt::TokenTree::Subtree(tt::Subtree { delimiter: None, token_trees })) } - _ => Ok(tt), + } else { + self.next().ok_or(()).cloned() } } fn expect_lifetime(&mut self) -> Result<tt::TokenTree, ()> { - let punct = self.expect_punct()?; + let punct = self.expect_single_punct()?; if punct.char != '\'' { return Err(()); } |