Unnamed repository; edit this file 'description' to name the repository.
Diffstat (limited to 'crates/mbe/src/expander/transcriber.rs')
| -rw-r--r-- | crates/mbe/src/expander/transcriber.rs | 423 |
1 files changed, 178 insertions, 245 deletions
diff --git a/crates/mbe/src/expander/transcriber.rs b/crates/mbe/src/expander/transcriber.rs index 1db2f35d26..68e9fad232 100644 --- a/crates/mbe/src/expander/transcriber.rs +++ b/crates/mbe/src/expander/transcriber.rs @@ -3,7 +3,7 @@ use intern::{sym, Symbol}; use span::{Edition, Span}; -use tt::Delimiter; +use tt::{iter::TtElement, Delimiter, TopSubtreeBuilder}; use crate::{ expander::{Binding, Bindings, Fragment}, @@ -11,8 +11,8 @@ use crate::{ ExpandError, ExpandErrorKind, ExpandResult, MetaTemplate, }; -impl Bindings { - fn get(&self, name: &Symbol, span: Span) -> Result<&Binding, ExpandError> { +impl<'t> Bindings<'t> { + fn get(&self, name: &Symbol, span: Span) -> Result<&Binding<'t>, ExpandError> { match self.inner.get(name) { Some(binding) => Ok(binding), None => Err(ExpandError::new( @@ -28,7 +28,7 @@ impl Bindings { mut span: Span, nesting: &mut [NestingState], marker: impl Fn(&mut Span), - ) -> Result<Fragment, ExpandError> { + ) -> Result<Fragment<'t>, ExpandError> { macro_rules! binding_err { ($($arg:tt)*) => { ExpandError::binding_error(span, format!($($arg)*)) }; } @@ -50,86 +50,61 @@ impl Bindings { }; } match b { - Binding::Fragment(f @ (Fragment::Path(sub) | Fragment::Expr(sub))) => { - let tt::Subtree { delimiter, token_trees } = sub; - marker(&mut span); - let subtree = tt::Subtree { - delimiter: tt::Delimiter { - // FIXME split span - open: span, - close: span, - kind: delimiter.kind, - }, - token_trees: token_trees.clone(), - }; - Ok(match f { - Fragment::Tokens(_) | Fragment::Empty => unreachable!(), - Fragment::Expr(_) => Fragment::Expr, - Fragment::Path(_) => Fragment::Path, - }(subtree)) - } - Binding::Fragment(it @ (Fragment::Tokens(_) | Fragment::Empty)) => Ok(it.clone()), + Binding::Fragment(f) => Ok(f.clone()), // emit some reasonable default expansion for missing bindings, // this gives better recovery than emitting the `$fragment-name` verbatim Binding::Missing(it) => Ok({ marker(&mut span); + let mut builder = TopSubtreeBuilder::new(tt::Delimiter::invisible_spanned(span)); match it { MetaVarKind::Stmt => { - Fragment::Tokens(tt::TokenTree::Leaf(tt::Leaf::Punct(tt::Punct { + builder.push(tt::Leaf::Punct(tt::Punct { span, char: ';', spacing: tt::Spacing::Alone, - }))) + })); } - MetaVarKind::Block => Fragment::Tokens(tt::TokenTree::Subtree(tt::Subtree { - delimiter: tt::Delimiter { - open: span, - close: span, - kind: tt::DelimiterKind::Brace, - }, - token_trees: Box::new([]), - })), - // FIXME: Meta and Item should get proper defaults - MetaVarKind::Meta | MetaVarKind::Item | MetaVarKind::Tt | MetaVarKind::Vis => { - Fragment::Empty + MetaVarKind::Block => { + builder.open(tt::DelimiterKind::Brace, span); + builder.close(span); } + // FIXME: Meta and Item should get proper defaults + MetaVarKind::Meta | MetaVarKind::Item | MetaVarKind::Tt | MetaVarKind::Vis => {} MetaVarKind::Path | MetaVarKind::Ty | MetaVarKind::Pat | MetaVarKind::PatParam | MetaVarKind::Expr(_) | MetaVarKind::Ident => { - Fragment::Tokens(tt::TokenTree::Leaf(tt::Leaf::Ident(tt::Ident { + builder.push(tt::Leaf::Ident(tt::Ident { sym: sym::missing.clone(), span, is_raw: tt::IdentIsRaw::No, - }))) + })); } MetaVarKind::Lifetime => { - Fragment::Tokens(tt::TokenTree::Subtree(tt::Subtree { - delimiter: tt::Delimiter::invisible_spanned(span), - token_trees: Box::new([ - tt::TokenTree::Leaf(tt::Leaf::Punct(tt::Punct { - char: '\'', - span, - spacing: tt::Spacing::Joint, - })), - tt::TokenTree::Leaf(tt::Leaf::Ident(tt::Ident { - sym: sym::missing.clone(), - span, - is_raw: tt::IdentIsRaw::No, - })), - ]), - })) + builder.extend([ + tt::Leaf::Punct(tt::Punct { + char: '\'', + span, + spacing: tt::Spacing::Joint, + }), + tt::Leaf::Ident(tt::Ident { + sym: sym::missing.clone(), + span, + is_raw: tt::IdentIsRaw::No, + }), + ]); } MetaVarKind::Literal => { - Fragment::Tokens(tt::TokenTree::Leaf(tt::Leaf::Ident(tt::Ident { + builder.push(tt::Leaf::Ident(tt::Ident { sym: sym::missing.clone(), span, is_raw: tt::IdentIsRaw::No, - }))) + })); } } + Fragment::TokensOwned(builder.build()) }), Binding::Nested(_) => { Err(binding_err!("expected simple binding, found nested binding `{name}`")) @@ -143,13 +118,13 @@ impl Bindings { pub(super) fn transcribe( template: &MetaTemplate, - bindings: &Bindings, + bindings: &Bindings<'_>, marker: impl Fn(&mut Span) + Copy, call_site: Span, -) -> ExpandResult<tt::Subtree<Span>> { +) -> ExpandResult<tt::TopSubtree<Span>> { let mut ctx = ExpandCtx { bindings, nesting: Vec::new(), call_site }; - let mut arena: Vec<tt::TokenTree<Span>> = Vec::new(); - expand_subtree(&mut ctx, template, None, &mut arena, marker) + let mut builder = tt::TopSubtreeBuilder::new(tt::Delimiter::invisible_spanned(ctx.call_site)); + expand_subtree(&mut ctx, template, &mut builder, marker).map(|()| builder.build()) } #[derive(Debug)] @@ -165,103 +140,97 @@ struct NestingState { #[derive(Debug)] struct ExpandCtx<'a> { - bindings: &'a Bindings, + bindings: &'a Bindings<'a>, nesting: Vec<NestingState>, call_site: Span, } -fn expand_subtree( +fn expand_subtree_with_delimiter( ctx: &mut ExpandCtx<'_>, template: &MetaTemplate, + builder: &mut tt::TopSubtreeBuilder<Span>, delimiter: Option<Delimiter<Span>>, - arena: &mut Vec<tt::TokenTree<Span>>, marker: impl Fn(&mut Span) + Copy, -) -> ExpandResult<tt::Subtree<Span>> { - // remember how many elements are in the arena now - when returning, we want to drain exactly how many elements we added. This way, the recursive uses of the arena get their own "view" of the arena, but will reuse the allocation - let start_elements = arena.len(); +) -> ExpandResult<()> { + let delimiter = delimiter.unwrap_or_else(|| tt::Delimiter::invisible_spanned(ctx.call_site)); + builder.open(delimiter.kind, delimiter.open); + let result = expand_subtree(ctx, template, builder, marker); + builder.close(delimiter.close); + result +} + +fn expand_subtree( + ctx: &mut ExpandCtx<'_>, + template: &MetaTemplate, + builder: &mut tt::TopSubtreeBuilder<Span>, + marker: impl Fn(&mut Span) + Copy, +) -> ExpandResult<()> { let mut err = None; 'ops: for op in template.iter() { match op { - Op::Literal(it) => arena.push( - tt::Leaf::from({ - let mut it = it.clone(); - marker(&mut it.span); - it - }) - .into(), - ), - Op::Ident(it) => arena.push( - tt::Leaf::from({ - let mut it = it.clone(); - marker(&mut it.span); - it - }) - .into(), - ), + Op::Literal(it) => builder.push(tt::Leaf::from({ + let mut it = it.clone(); + marker(&mut it.span); + it + })), + Op::Ident(it) => builder.push(tt::Leaf::from({ + let mut it = it.clone(); + marker(&mut it.span); + it + })), Op::Punct(puncts) => { - for punct in puncts.as_slice() { - arena.push( - tt::Leaf::from({ - let mut it = *punct; - marker(&mut it.span); - it - }) - .into(), - ); - } + builder.extend(puncts.iter().map(|punct| { + tt::Leaf::from({ + let mut it = *punct; + marker(&mut it.span); + it + }) + })); } Op::Subtree { tokens, delimiter } => { let mut delimiter = *delimiter; marker(&mut delimiter.open); marker(&mut delimiter.close); - let ExpandResult { value: tt, err: e } = - expand_subtree(ctx, tokens, Some(delimiter), arena, marker); + let ExpandResult { value: (), err: e } = + expand_subtree_with_delimiter(ctx, tokens, builder, Some(delimiter), marker); err = err.or(e); - arena.push(tt.into()); } Op::Var { name, id, .. } => { - let ExpandResult { value: fragment, err: e } = expand_var(ctx, name, *id, marker); + let ExpandResult { value: (), err: e } = + expand_var(ctx, name, *id, builder, marker); err = err.or(e); - push_fragment(ctx, arena, fragment); } Op::Repeat { tokens: subtree, kind, separator } => { - let ExpandResult { value: fragment, err: e } = - expand_repeat(ctx, subtree, *kind, separator.as_deref(), arena, marker); + let ExpandResult { value: (), err: e } = + expand_repeat(ctx, subtree, *kind, separator.as_deref(), builder, marker); err = err.or(e); - push_fragment(ctx, arena, fragment) } Op::Ignore { name, id } => { // Expand the variable, but ignore the result. This registers the repetition count. // FIXME: Any emitted errors are dropped. - expand_var(ctx, name, *id, marker); + let _ = ctx.bindings.get_fragment(name, *id, &mut ctx.nesting, marker); } Op::Index { depth } => { let index = ctx.nesting.get(ctx.nesting.len() - 1 - depth).map_or(0, |nest| nest.idx); - arena.push( - tt::Leaf::Literal(tt::Literal { - symbol: Symbol::integer(index), - span: ctx.call_site, - kind: tt::LitKind::Integer, - suffix: None, - }) - .into(), - ); + builder.push(tt::Leaf::Literal(tt::Literal { + symbol: Symbol::integer(index), + span: ctx.call_site, + kind: tt::LitKind::Integer, + suffix: None, + })); } Op::Len { depth } => { let length = ctx.nesting.get(ctx.nesting.len() - 1 - depth).map_or(0, |_nest| { // FIXME: to be implemented 0 }); - arena.push( - tt::Leaf::Literal(tt::Literal { - symbol: Symbol::integer(length), - span: ctx.call_site, - kind: tt::LitKind::Integer, - suffix: None, - }) - .into(), - ); + builder.push(tt::Leaf::Literal(tt::Literal { + symbol: Symbol::integer(length), + span: ctx.call_site, + kind: tt::LitKind::Integer, + suffix: None, + })); } Op::Count { name, depth } => { let mut binding = match ctx.bindings.get(name, ctx.call_site) { @@ -302,15 +271,12 @@ fn expand_subtree( let res = count(binding, 0, depth.unwrap_or(0)); - arena.push( - tt::Leaf::Literal(tt::Literal { - symbol: Symbol::integer(res), - span: ctx.call_site, - suffix: None, - kind: tt::LitKind::Integer, - }) - .into(), - ); + builder.push(tt::Leaf::Literal(tt::Literal { + symbol: Symbol::integer(res), + span: ctx.call_site, + suffix: None, + kind: tt::LitKind::Integer, + })); } Op::Concat { elements, span: concat_span } => { let mut concatenated = String::new(); @@ -342,11 +308,22 @@ fn expand_subtree( continue; } }; - let value = match &var_value { - Fragment::Tokens(tt::TokenTree::Leaf(tt::Leaf::Ident(ident))) => { + let values = match &var_value { + Fragment::Tokens(tokens) => { + let mut iter = tokens.iter(); + (iter.next(), iter.next()) + } + Fragment::TokensOwned(tokens) => { + let mut iter = tokens.iter(); + (iter.next(), iter.next()) + } + _ => (None, None), + }; + let value = match values { + (Some(TtElement::Leaf(tt::Leaf::Ident(ident))), None) => { ident.sym.as_str() } - Fragment::Tokens(tt::TokenTree::Leaf(tt::Leaf::Literal(lit))) => { + (Some(TtElement::Leaf(tt::Leaf::Literal(lit))), None) => { lit.symbol.as_str() } _ => { @@ -382,36 +359,53 @@ fn expand_subtree( let needs_raw = parser::SyntaxKind::from_keyword(&concatenated, Edition::LATEST).is_some(); let is_raw = if needs_raw { tt::IdentIsRaw::Yes } else { tt::IdentIsRaw::No }; - arena.push(tt::TokenTree::Leaf(tt::Leaf::Ident(tt::Ident { + builder.push(tt::Leaf::Ident(tt::Ident { is_raw, span: result_span, sym: Symbol::intern(&concatenated), - }))); + })); } } } - // drain the elements added in this instance of expand_subtree - let tts = arena.drain(start_elements..).collect(); - ExpandResult { - value: tt::Subtree { - delimiter: delimiter.unwrap_or_else(|| tt::Delimiter::invisible_spanned(ctx.call_site)), - token_trees: tts, - }, - err, - } + ExpandResult { value: (), err } } fn expand_var( ctx: &mut ExpandCtx<'_>, v: &Symbol, id: Span, - marker: impl Fn(&mut Span), -) -> ExpandResult<Fragment> { + builder: &mut tt::TopSubtreeBuilder<Span>, + marker: impl Fn(&mut Span) + Copy, +) -> ExpandResult<()> { // We already handle $crate case in mbe parser debug_assert!(*v != sym::crate_); match ctx.bindings.get_fragment(v, id, &mut ctx.nesting, marker) { - Ok(it) => ExpandResult::ok(it), + Ok(fragment) => { + match fragment { + Fragment::Tokens(tt) => builder.extend_with_tt(tt.strip_invisible()), + Fragment::TokensOwned(tt) => builder.extend_with_tt(tt.view().strip_invisible()), + Fragment::Expr(sub) => { + let sub = sub.strip_invisible(); + let mut span = id; + marker(&mut span); + let wrap_in_parens = !matches!(sub.flat_tokens(), [tt::TokenTree::Leaf(_)]) + && sub.try_into_subtree().map_or(true, |it| { + it.top_subtree().delimiter.kind == tt::DelimiterKind::Invisible + }); + if wrap_in_parens { + builder.open(tt::DelimiterKind::Parenthesis, span); + } + builder.extend_with_tt(sub); + if wrap_in_parens { + builder.close(span); + } + } + Fragment::Path(tt) => fix_up_and_push_path_tt(ctx, builder, tt), + Fragment::Empty => (), + }; + ExpandResult::ok(()) + } Err(e) if matches!(e.inner.1, ExpandErrorKind::UnresolvedBinding(_)) => { // Note that it is possible to have a `$var` inside a macro which is not bound. // For example: @@ -426,29 +420,13 @@ fn expand_var( // } // ``` // We just treat it a normal tokens - let tt = tt::Subtree { - delimiter: tt::Delimiter::invisible_spanned(id), - token_trees: Box::new([ - tt::Leaf::from(tt::Punct { char: '$', spacing: tt::Spacing::Alone, span: id }) - .into(), - tt::Leaf::from(tt::Ident { - sym: v.clone(), - span: id, - is_raw: tt::IdentIsRaw::No, - }) - .into(), - ]), - } - .into(); - ExpandResult::ok(Fragment::Tokens(tt)) + builder.extend([ + tt::Leaf::from(tt::Punct { char: '$', spacing: tt::Spacing::Alone, span: id }), + tt::Leaf::from(tt::Ident { sym: v.clone(), span: id, is_raw: tt::IdentIsRaw::No }), + ]); + ExpandResult::ok(()) } - Err(e) => ExpandResult { - value: Fragment::Tokens(tt::TokenTree::Subtree(tt::Subtree::empty(tt::DelimSpan { - open: ctx.call_site, - close: ctx.call_site, - }))), - err: Some(e), - }, + Err(e) => ExpandResult::only_err(e), } } @@ -457,21 +435,20 @@ fn expand_repeat( template: &MetaTemplate, kind: RepeatKind, separator: Option<&Separator>, - arena: &mut Vec<tt::TokenTree<Span>>, + builder: &mut tt::TopSubtreeBuilder<Span>, marker: impl Fn(&mut Span) + Copy, -) -> ExpandResult<Fragment> { - let mut buf: Vec<tt::TokenTree<Span>> = Vec::new(); +) -> ExpandResult<()> { ctx.nesting.push(NestingState { idx: 0, at_end: false, hit: false }); // Dirty hack to make macro-expansion terminate. // This should be replaced by a proper macro-by-example implementation let limit = 65536; - let mut has_seps = 0; let mut counter = 0; let mut err = None; + let mut restore_point = builder.restore_point(); loop { - let ExpandResult { value: mut t, err: e } = - expand_subtree(ctx, template, None, arena, marker); + let ExpandResult { value: (), err: e } = + expand_subtree_with_delimiter(ctx, template, builder, None, marker); let nesting_state = ctx.nesting.last_mut().unwrap(); if nesting_state.at_end || !nesting_state.hit { break; @@ -479,6 +456,10 @@ fn expand_repeat( nesting_state.idx += 1; nesting_state.hit = false; + builder.remove_last_subtree_if_invisible(); + + restore_point = builder.restore_point(); + counter += 1; if counter == limit { tracing::warn!( @@ -486,16 +467,8 @@ fn expand_repeat( template, ctx ); - return ExpandResult { - value: Fragment::Tokens( - tt::Subtree { - delimiter: tt::Delimiter::invisible_spanned(ctx.call_site), - token_trees: Box::new([]), - } - .into(), - ), - err: Some(ExpandError::new(ctx.call_site, ExpandErrorKind::LimitExceeded)), - }; + err = Some(ExpandError::new(ctx.call_site, ExpandErrorKind::LimitExceeded)); + break; } if e.is_some() { @@ -503,24 +476,14 @@ fn expand_repeat( continue; } - t.delimiter.kind = tt::DelimiterKind::Invisible; - push_subtree(&mut buf, t); - if let Some(sep) = separator { - has_seps = match sep { - Separator::Ident(ident) => { - buf.push(tt::Leaf::from(ident.clone()).into()); - 1 - } - Separator::Literal(lit) => { - buf.push(tt::Leaf::from(lit.clone()).into()); - 1 - } + match sep { + Separator::Ident(ident) => builder.push(tt::Leaf::from(ident.clone())), + Separator::Literal(lit) => builder.push(tt::Leaf::from(lit.clone())), Separator::Puncts(puncts) => { for &punct in puncts { - buf.push(tt::Leaf::from(punct).into()); + builder.push(tt::Leaf::from(punct)); } - puncts.len() } }; } @@ -529,46 +492,18 @@ fn expand_repeat( break; } } + // Lose the last separator and last after-the-end round. + builder.restore(restore_point); ctx.nesting.pop().unwrap(); - for _ in 0..has_seps { - buf.pop(); - } // Check if it is a single token subtree without any delimiter // e.g {Delimiter:None> ['>'] /Delimiter:None>} - let tt = tt::Subtree { - delimiter: tt::Delimiter::invisible_spanned(ctx.call_site), - token_trees: buf.into_boxed_slice(), - }; - - if RepeatKind::OneOrMore == kind && counter == 0 { - let span = tt.delimiter.open; - return ExpandResult { - value: Fragment::Tokens(tt.into()), - err: Some(ExpandError::new(span, ExpandErrorKind::UnexpectedToken)), - }; - } - ExpandResult { value: Fragment::Tokens(tt.into()), err } -} -fn push_fragment(ctx: &ExpandCtx<'_>, buf: &mut Vec<tt::TokenTree<Span>>, fragment: Fragment) { - match fragment { - Fragment::Tokens(tt::TokenTree::Subtree(tt)) => push_subtree(buf, tt), - Fragment::Expr(sub) => { - push_subtree(buf, sub); - } - Fragment::Path(tt) => fix_up_and_push_path_tt(ctx, buf, tt), - Fragment::Tokens(tt) => buf.push(tt), - Fragment::Empty => (), - } -} - -fn push_subtree(buf: &mut Vec<tt::TokenTree<Span>>, tt: tt::Subtree<Span>) { - match tt.delimiter.kind { - tt::DelimiterKind::Invisible => buf.extend(Vec::from(tt.token_trees)), - _ => buf.push(tt.into()), + if RepeatKind::OneOrMore == kind && counter == 0 && err.is_none() { + err = Some(ExpandError::new(ctx.call_site, ExpandErrorKind::UnexpectedToken)); } + ExpandResult { value: (), err } } /// Inserts the path separator `::` between an identifier and its following generic @@ -576,47 +511,45 @@ fn push_subtree(buf: &mut Vec<tt::TokenTree<Span>>, tt: tt::Subtree<Span>) { /// we need this fixup. fn fix_up_and_push_path_tt( ctx: &ExpandCtx<'_>, - buf: &mut Vec<tt::TokenTree<Span>>, - subtree: tt::Subtree<Span>, + builder: &mut tt::TopSubtreeBuilder<Span>, + subtree: tt::TokenTreesView<'_, Span>, ) { - stdx::always!(matches!(subtree.delimiter.kind, tt::DelimiterKind::Invisible)); let mut prev_was_ident = false; // Note that we only need to fix up the top-level `TokenTree`s because the // context of the paths in the descendant `Subtree`s won't be changed by the // mbe transcription. - for tt in Vec::from(subtree.token_trees) { + let mut iter = subtree.iter(); + while let Some(tt) = iter.next_as_view() { if prev_was_ident { // Pedantically, `(T) -> U` in `FnOnce(T) -> U` is treated as a generic // argument list and thus needs `::` between it and `FnOnce`. However in // today's Rust this type of path *semantically* cannot appear as a // top-level expression-context path, so we can safely ignore it. - if let tt::TokenTree::Leaf(tt::Leaf::Punct(tt::Punct { char: '<', .. })) = tt { - buf.push( + if let [tt::TokenTree::Leaf(tt::Leaf::Punct(tt::Punct { char: '<', .. }))] = + tt.flat_tokens() + { + builder.extend([ tt::Leaf::Punct(tt::Punct { char: ':', spacing: tt::Spacing::Joint, span: ctx.call_site, - }) - .into(), - ); - buf.push( + }), tt::Leaf::Punct(tt::Punct { char: ':', spacing: tt::Spacing::Alone, span: ctx.call_site, - }) - .into(), - ); + }), + ]); } } - prev_was_ident = matches!(tt, tt::TokenTree::Leaf(tt::Leaf::Ident(_))); - buf.push(tt); + prev_was_ident = matches!(tt.flat_tokens(), [tt::TokenTree::Leaf(tt::Leaf::Ident(_))]); + builder.extend_with_tt(tt); } } /// Handles `${count(t, depth)}`. `our_depth` is the recursion depth and `count_depth` is the depth /// defined by the metavar expression. -fn count(binding: &Binding, depth_curr: usize, depth_max: usize) -> usize { +fn count(binding: &Binding<'_>, depth_curr: usize, depth_max: usize) -> usize { match binding { Binding::Nested(bs) => { if depth_curr == depth_max { |