Unnamed repository; edit this file 'description' to name the repository.
Diffstat (limited to 'crates/mbe/src/expander/transcriber.rs')
| -rw-r--r-- | crates/mbe/src/expander/transcriber.rs | 338 |
1 files changed, 197 insertions, 141 deletions
diff --git a/crates/mbe/src/expander/transcriber.rs b/crates/mbe/src/expander/transcriber.rs index cdac2f1e3b..7a3e8653c2 100644 --- a/crates/mbe/src/expander/transcriber.rs +++ b/crates/mbe/src/expander/transcriber.rs @@ -2,31 +2,29 @@ //! `$ident => foo`, interpolates variables in the template, to get `fn foo() {}` use syntax::SmolStr; +use tt::{Delimiter, Span}; use crate::{ expander::{Binding, Bindings, Fragment}, parser::{MetaVarKind, Op, RepeatKind, Separator}, - tt::{self, Delimiter}, CountError, ExpandError, ExpandResult, MetaTemplate, }; -impl Bindings { - fn contains(&self, name: &str) -> bool { - self.inner.contains_key(name) - } - - fn get(&self, name: &str) -> Result<&Binding, ExpandError> { +impl<S: Span> Bindings<S> { + fn get(&self, name: &str) -> Result<&Binding<S>, ExpandError> { match self.inner.get(name) { Some(binding) => Ok(binding), - None => Err(ExpandError::binding_error(format!("could not find binding `{name}`"))), + None => Err(ExpandError::UnresolvedBinding(Box::new(Box::from(name)))), } } fn get_fragment( &self, name: &str, + mut span: S, nesting: &mut [NestingState], - ) -> Result<Fragment, ExpandError> { + marker: impl Fn(&mut S), + ) -> Result<Fragment<S>, ExpandError> { macro_rules! binding_err { ($($arg:tt)*) => { ExpandError::binding_error(format!($($arg)*)) }; } @@ -48,54 +46,75 @@ impl Bindings { }; } match b { - Binding::Fragment(it) => Ok(it.clone()), - // emit some reasonable default expansion for missing bindings, - // this gives better recovery than emitting the `$fragment-name` verbatim - Binding::Missing(it) => Ok(match it { - MetaVarKind::Stmt => { - Fragment::Tokens(tt::TokenTree::Leaf(tt::Leaf::Punct(tt::Punct { - span: tt::TokenId::unspecified(), - char: ';', - spacing: tt::Spacing::Alone, - }))) - } - MetaVarKind::Block => Fragment::Tokens(tt::TokenTree::Subtree(tt::Subtree { + Binding::Fragment(f @ (Fragment::Path(sub) | Fragment::Expr(sub))) => { + let tt::Subtree { delimiter, token_trees } = sub; + marker(&mut span); + let subtree = tt::Subtree { delimiter: tt::Delimiter { - open: tt::TokenId::unspecified(), - close: tt::TokenId::unspecified(), - kind: tt::DelimiterKind::Brace, + // FIXME split span + open: span, + close: span, + kind: delimiter.kind, }, - token_trees: vec![], - })), - // FIXME: Meta and Item should get proper defaults - MetaVarKind::Meta | MetaVarKind::Item | MetaVarKind::Tt | MetaVarKind::Vis => { - Fragment::Tokens(tt::TokenTree::Subtree(tt::Subtree { - delimiter: tt::Delimiter::UNSPECIFIED, + token_trees: token_trees.clone(), + }; + Ok(match f { + Fragment::Tokens(_) => unreachable!(), + Fragment::Expr(_) => Fragment::Expr, + Fragment::Path(_) => Fragment::Path, + }(subtree)) + } + Binding::Fragment(it @ Fragment::Tokens(_)) => Ok(it.clone()), + // emit some reasonable default expansion for missing bindings, + // this gives better recovery than emitting the `$fragment-name` verbatim + Binding::Missing(it) => Ok({ + marker(&mut span); + match it { + MetaVarKind::Stmt => { + Fragment::Tokens(tt::TokenTree::Leaf(tt::Leaf::Punct(tt::Punct { + span, + char: ';', + spacing: tt::Spacing::Alone, + }))) + } + MetaVarKind::Block => Fragment::Tokens(tt::TokenTree::Subtree(tt::Subtree { + delimiter: tt::Delimiter { + open: span, + close: span, + kind: tt::DelimiterKind::Brace, + }, token_trees: vec![], - })) - } - MetaVarKind::Path - | MetaVarKind::Ty - | MetaVarKind::Pat - | MetaVarKind::PatParam - | MetaVarKind::Expr - | MetaVarKind::Ident => { - Fragment::Tokens(tt::TokenTree::Leaf(tt::Leaf::Ident(tt::Ident { - text: SmolStr::new_inline("missing"), - span: tt::TokenId::unspecified(), - }))) - } - MetaVarKind::Lifetime => { - Fragment::Tokens(tt::TokenTree::Leaf(tt::Leaf::Ident(tt::Ident { - text: SmolStr::new_inline("'missing"), - span: tt::TokenId::unspecified(), - }))) - } - MetaVarKind::Literal => { - Fragment::Tokens(tt::TokenTree::Leaf(tt::Leaf::Ident(tt::Ident { - text: SmolStr::new_inline("\"missing\""), - span: tt::TokenId::unspecified(), - }))) + })), + // FIXME: Meta and Item should get proper defaults + MetaVarKind::Meta | MetaVarKind::Item | MetaVarKind::Tt | MetaVarKind::Vis => { + Fragment::Tokens(tt::TokenTree::Subtree(tt::Subtree { + delimiter: tt::Delimiter::DUMMY_INVISIBLE, + token_trees: vec![], + })) + } + MetaVarKind::Path + | MetaVarKind::Ty + | MetaVarKind::Pat + | MetaVarKind::PatParam + | MetaVarKind::Expr + | MetaVarKind::Ident => { + Fragment::Tokens(tt::TokenTree::Leaf(tt::Leaf::Ident(tt::Ident { + text: SmolStr::new_inline("missing"), + span, + }))) + } + MetaVarKind::Lifetime => { + Fragment::Tokens(tt::TokenTree::Leaf(tt::Leaf::Ident(tt::Ident { + text: SmolStr::new_inline("'missing"), + span, + }))) + } + MetaVarKind::Literal => { + Fragment::Tokens(tt::TokenTree::Leaf(tt::Leaf::Ident(tt::Ident { + text: SmolStr::new_inline("\"missing\""), + span, + }))) + } } }), Binding::Nested(_) => { @@ -108,13 +127,14 @@ impl Bindings { } } -pub(super) fn transcribe( - template: &MetaTemplate, - bindings: &Bindings, -) -> ExpandResult<tt::Subtree> { +pub(super) fn transcribe<S: Span>( + template: &MetaTemplate<S>, + bindings: &Bindings<S>, + marker: impl Fn(&mut S) + Copy, +) -> ExpandResult<tt::Subtree<S>> { let mut ctx = ExpandCtx { bindings, nesting: Vec::new() }; - let mut arena: Vec<tt::TokenTree> = Vec::new(); - expand_subtree(&mut ctx, template, None, &mut arena) + let mut arena: Vec<tt::TokenTree<S>> = Vec::new(); + expand_subtree(&mut ctx, template, None, &mut arena, marker) } #[derive(Debug)] @@ -129,50 +149,75 @@ struct NestingState { } #[derive(Debug)] -struct ExpandCtx<'a> { - bindings: &'a Bindings, +struct ExpandCtx<'a, S> { + bindings: &'a Bindings<S>, nesting: Vec<NestingState>, } -fn expand_subtree( - ctx: &mut ExpandCtx<'_>, - template: &MetaTemplate, - delimiter: Option<Delimiter>, - arena: &mut Vec<tt::TokenTree>, -) -> ExpandResult<tt::Subtree> { +fn expand_subtree<S: Span>( + ctx: &mut ExpandCtx<'_, S>, + template: &MetaTemplate<S>, + delimiter: Option<Delimiter<S>>, + arena: &mut Vec<tt::TokenTree<S>>, + marker: impl Fn(&mut S) + Copy, +) -> ExpandResult<tt::Subtree<S>> { // remember how many elements are in the arena now - when returning, we want to drain exactly how many elements we added. This way, the recursive uses of the arena get their own "view" of the arena, but will reuse the allocation let start_elements = arena.len(); let mut err = None; 'ops: for op in template.iter() { match op { - Op::Literal(it) => arena.push(tt::Leaf::from(it.clone()).into()), - Op::Ident(it) => arena.push(tt::Leaf::from(it.clone()).into()), + Op::Literal(it) => arena.push( + tt::Leaf::from({ + let mut it = it.clone(); + marker(&mut it.span); + it + }) + .into(), + ), + Op::Ident(it) => arena.push( + tt::Leaf::from({ + let mut it = it.clone(); + marker(&mut it.span); + it + }) + .into(), + ), Op::Punct(puncts) => { for punct in puncts { - arena.push(tt::Leaf::from(*punct).into()); + arena.push( + tt::Leaf::from({ + let mut it = punct.clone(); + marker(&mut it.span); + it + }) + .into(), + ); } } Op::Subtree { tokens, delimiter } => { + let mut delimiter = *delimiter; + marker(&mut delimiter.open); + marker(&mut delimiter.close); let ExpandResult { value: tt, err: e } = - expand_subtree(ctx, tokens, Some(*delimiter), arena); + expand_subtree(ctx, tokens, Some(delimiter), arena, marker); err = err.or(e); arena.push(tt.into()); } Op::Var { name, id, .. } => { - let ExpandResult { value: fragment, err: e } = expand_var(ctx, name, *id); + let ExpandResult { value: fragment, err: e } = expand_var(ctx, name, *id, marker); err = err.or(e); push_fragment(arena, fragment); } Op::Repeat { tokens: subtree, kind, separator } => { let ExpandResult { value: fragment, err: e } = - expand_repeat(ctx, subtree, *kind, separator, arena); + expand_repeat(ctx, subtree, *kind, separator, arena, marker); err = err.or(e); push_fragment(arena, fragment) } Op::Ignore { name, id } => { // Expand the variable, but ignore the result. This registers the repetition count. // FIXME: Any emitted errors are dropped. - expand_var(ctx, name, *id); + expand_var(ctx, name, *id, marker); } Op::Index { depth } => { let index = @@ -180,7 +225,8 @@ fn expand_subtree( arena.push( tt::Leaf::Literal(tt::Literal { text: index.to_string().into(), - span: tt::TokenId::unspecified(), + // FIXME + span: S::DUMMY, }) .into(), ); @@ -239,7 +285,8 @@ fn expand_subtree( arena.push( tt::Leaf::Literal(tt::Literal { text: c.to_string().into(), - span: tt::TokenId::unspecified(), + // FIXME + span: S::DUMMY, }) .into(), ); @@ -250,60 +297,70 @@ fn expand_subtree( let tts = arena.drain(start_elements..).collect(); ExpandResult { value: tt::Subtree { - delimiter: delimiter.unwrap_or_else(tt::Delimiter::unspecified), + delimiter: delimiter.unwrap_or_else(tt::Delimiter::dummy_invisible), token_trees: tts, }, err, } } -fn expand_var(ctx: &mut ExpandCtx<'_>, v: &SmolStr, id: tt::TokenId) -> ExpandResult<Fragment> { +fn expand_var<S: Span>( + ctx: &mut ExpandCtx<'_, S>, + v: &SmolStr, + id: S, + marker: impl Fn(&mut S), +) -> ExpandResult<Fragment<S>> { // We already handle $crate case in mbe parser debug_assert!(v != "crate"); - if !ctx.bindings.contains(v) { - // Note that it is possible to have a `$var` inside a macro which is not bound. - // For example: - // ``` - // macro_rules! foo { - // ($a:ident, $b:ident, $c:tt) => { - // macro_rules! bar { - // ($bi:ident) => { - // fn $bi() -> u8 {$c} - // } - // } - // } - // ``` - // We just treat it a normal tokens - let tt = tt::Subtree { - delimiter: tt::Delimiter::UNSPECIFIED, - token_trees: vec![ - tt::Leaf::from(tt::Punct { char: '$', spacing: tt::Spacing::Alone, span: id }) - .into(), - tt::Leaf::from(tt::Ident { text: v.clone(), span: id }).into(), - ], + match ctx.bindings.get_fragment(v, id, &mut ctx.nesting, marker) { + Ok(it) => ExpandResult::ok(it), + Err(ExpandError::UnresolvedBinding(_)) => { + // Note that it is possible to have a `$var` inside a macro which is not bound. + // For example: + // ``` + // macro_rules! foo { + // ($a:ident, $b:ident, $c:tt) => { + // macro_rules! bar { + // ($bi:ident) => { + // fn $bi() -> u8 {$c} + // } + // } + // } + // ``` + // We just treat it a normal tokens + let tt = tt::Subtree { + delimiter: tt::Delimiter::DUMMY_INVISIBLE, + token_trees: vec![ + tt::Leaf::from(tt::Punct { char: '$', spacing: tt::Spacing::Alone, span: id }) + .into(), + tt::Leaf::from(tt::Ident { text: v.clone(), span: id }).into(), + ], + } + .into(); + ExpandResult::ok(Fragment::Tokens(tt)) } - .into(); - ExpandResult::ok(Fragment::Tokens(tt)) - } else { - ctx.bindings.get_fragment(v, &mut ctx.nesting).map_or_else( - |e| ExpandResult { - value: Fragment::Tokens(tt::TokenTree::Subtree(tt::Subtree::empty())), - err: Some(e), - }, - ExpandResult::ok, - ) + Err(e) => ExpandResult { + value: Fragment::Tokens(tt::TokenTree::Subtree(tt::Subtree::empty(tt::DelimSpan { + // FIXME + open: S::DUMMY, + // FIXME + close: S::DUMMY, + }))), + err: Some(e), + }, } } -fn expand_repeat( - ctx: &mut ExpandCtx<'_>, - template: &MetaTemplate, +fn expand_repeat<S: Span>( + ctx: &mut ExpandCtx<'_, S>, + template: &MetaTemplate<S>, kind: RepeatKind, - separator: &Option<Separator>, - arena: &mut Vec<tt::TokenTree>, -) -> ExpandResult<Fragment> { - let mut buf: Vec<tt::TokenTree> = Vec::new(); + separator: &Option<Separator<S>>, + arena: &mut Vec<tt::TokenTree<S>>, + marker: impl Fn(&mut S) + Copy, +) -> ExpandResult<Fragment<S>> { + let mut buf: Vec<tt::TokenTree<S>> = Vec::new(); ctx.nesting.push(NestingState { idx: 0, at_end: false, hit: false }); // Dirty hack to make macro-expansion terminate. // This should be replaced by a proper macro-by-example implementation @@ -313,7 +370,8 @@ fn expand_repeat( let mut err = None; loop { - let ExpandResult { value: mut t, err: e } = expand_subtree(ctx, template, None, arena); + let ExpandResult { value: mut t, err: e } = + expand_subtree(ctx, template, None, arena, marker); let nesting_state = ctx.nesting.last_mut().unwrap(); if nesting_state.at_end || !nesting_state.hit { break; @@ -330,8 +388,11 @@ fn expand_repeat( ); return ExpandResult { value: Fragment::Tokens( - tt::Subtree { delimiter: tt::Delimiter::unspecified(), token_trees: vec![] } - .into(), + tt::Subtree { + delimiter: tt::Delimiter::dummy_invisible(), + token_trees: vec![], + } + .into(), ), err: Some(ExpandError::LimitExceeded), }; @@ -342,7 +403,7 @@ fn expand_repeat( continue; } - t.delimiter = tt::Delimiter::unspecified(); + t.delimiter = tt::Delimiter::DUMMY_INVISIBLE; push_subtree(&mut buf, t); if let Some(sep) = separator { @@ -376,7 +437,7 @@ fn expand_repeat( // Check if it is a single token subtree without any delimiter // e.g {Delimiter:None> ['>'] /Delimiter:None>} - let tt = tt::Subtree { delimiter: tt::Delimiter::unspecified(), token_trees: buf }.into(); + let tt = tt::Subtree { delimiter: tt::Delimiter::DUMMY_INVISIBLE, token_trees: buf }.into(); if RepeatKind::OneOrMore == kind && counter == 0 { return ExpandResult { @@ -387,25 +448,18 @@ fn expand_repeat( ExpandResult { value: Fragment::Tokens(tt), err } } -fn push_fragment(buf: &mut Vec<tt::TokenTree>, fragment: Fragment) { +fn push_fragment<S: Span>(buf: &mut Vec<tt::TokenTree<S>>, fragment: Fragment<S>) { match fragment { Fragment::Tokens(tt::TokenTree::Subtree(tt)) => push_subtree(buf, tt), - Fragment::Expr(tt::TokenTree::Subtree(mut tt)) => { - if tt.delimiter.kind == tt::DelimiterKind::Invisible { - tt.delimiter = tt::Delimiter { - open: tt::TokenId::UNSPECIFIED, - close: tt::TokenId::UNSPECIFIED, - kind: tt::DelimiterKind::Parenthesis, - }; - } - buf.push(tt.into()) + Fragment::Expr(sub) => { + push_subtree(buf, sub); } - Fragment::Path(tt::TokenTree::Subtree(tt)) => fix_up_and_push_path_tt(buf, tt), - Fragment::Tokens(tt) | Fragment::Expr(tt) | Fragment::Path(tt) => buf.push(tt), + Fragment::Path(tt) => fix_up_and_push_path_tt(buf, tt), + Fragment::Tokens(tt) => buf.push(tt), } } -fn push_subtree(buf: &mut Vec<tt::TokenTree>, tt: tt::Subtree) { +fn push_subtree<S>(buf: &mut Vec<tt::TokenTree<S>>, tt: tt::Subtree<S>) { match tt.delimiter.kind { tt::DelimiterKind::Invisible => buf.extend(tt.token_trees), _ => buf.push(tt.into()), @@ -415,7 +469,7 @@ fn push_subtree(buf: &mut Vec<tt::TokenTree>, tt: tt::Subtree) { /// Inserts the path separator `::` between an identifier and its following generic /// argument list, and then pushes into the buffer. See [`Fragment::Path`] for why /// we need this fixup. -fn fix_up_and_push_path_tt(buf: &mut Vec<tt::TokenTree>, subtree: tt::Subtree) { +fn fix_up_and_push_path_tt<S: Span>(buf: &mut Vec<tt::TokenTree<S>>, subtree: tt::Subtree<S>) { stdx::always!(matches!(subtree.delimiter.kind, tt::DelimiterKind::Invisible)); let mut prev_was_ident = false; // Note that we only need to fix up the top-level `TokenTree`s because the @@ -432,7 +486,8 @@ fn fix_up_and_push_path_tt(buf: &mut Vec<tt::TokenTree>, subtree: tt::Subtree) { tt::Leaf::Punct(tt::Punct { char: ':', spacing: tt::Spacing::Joint, - span: tt::Span::unspecified(), + // FIXME + span: S::DUMMY, }) .into(), ); @@ -440,7 +495,8 @@ fn fix_up_and_push_path_tt(buf: &mut Vec<tt::TokenTree>, subtree: tt::Subtree) { tt::Leaf::Punct(tt::Punct { char: ':', spacing: tt::Spacing::Alone, - span: tt::Span::unspecified(), + // FIXME + span: S::DUMMY, }) .into(), ); @@ -453,9 +509,9 @@ fn fix_up_and_push_path_tt(buf: &mut Vec<tt::TokenTree>, subtree: tt::Subtree) { /// Handles `${count(t, depth)}`. `our_depth` is the recursion depth and `count_depth` is the depth /// defined by the metavar expression. -fn count( - ctx: &ExpandCtx<'_>, - binding: &Binding, +fn count<S>( + ctx: &ExpandCtx<'_, S>, + binding: &Binding<S>, our_depth: usize, count_depth: Option<usize>, ) -> Result<usize, CountError> { |