Unnamed repository; edit this file 'description' to name the repository.
24 files changed, 350 insertions, 293 deletions
diff --git a/crates/cfg/src/cfg_expr.rs b/crates/cfg/src/cfg_expr.rs index 2f1fc8ae49..c5da8443a6 100644 --- a/crates/cfg/src/cfg_expr.rs +++ b/crates/cfg/src/cfg_expr.rs @@ -197,17 +197,18 @@ fn next_cfg_expr(it: &mut tt::iter::TtIter<'_>) -> Option<CfgExpr> { Some(_) => return Some(CfgExpr::Invalid), }; - let ret = match it.peek() { + let mut it_clone = it.clone(); + let ret = match it_clone.next() { Some(TtElement::Leaf(tt::Leaf::Punct(punct))) // Don't consume on e.g. `=>`. if punct.char == '=' && (punct.spacing == tt::Spacing::Alone - || it.remaining().flat_tokens().get(1).is_none_or(|peek2| { - !matches!(peek2, tt::TokenTree::Leaf(tt::Leaf::Punct(_))) + || it_clone.peek().is_none_or(|peek2| { + !matches!(peek2, tt::TtElement::Leaf(tt::Leaf::Punct(_))) })) => { - match it.remaining().flat_tokens().get(1) { - Some(tt::TokenTree::Leaf(tt::Leaf::Literal(literal))) => { + match it_clone.next() { + Some(tt::TtElement::Leaf(tt::Leaf::Literal(literal))) => { it.next(); it.next(); CfgAtom::KeyValue { key: name, value: literal.symbol.clone() }.into() diff --git a/crates/hir-def/src/nameres/attr_resolution.rs b/crates/hir-def/src/nameres/attr_resolution.rs index 1cbd2c10b5..062b55fcef 100644 --- a/crates/hir-def/src/nameres/attr_resolution.rs +++ b/crates/hir-def/src/nameres/attr_resolution.rs @@ -114,7 +114,7 @@ pub(super) fn attr_macro_as_call_id( let arg = match macro_attr.input.as_deref() { Some(AttrInput::TokenTree(tt)) => { let mut tt = tt.clone(); - tt.top_subtree_delimiter_mut().kind = tt::DelimiterKind::Invisible; + tt.set_top_subtree_delimiter_kind(tt::DelimiterKind::Invisible); Some(tt) } diff --git a/crates/hir-def/src/nameres/collector.rs b/crates/hir-def/src/nameres/collector.rs index a7f687a316..822da678d7 100644 --- a/crates/hir-def/src/nameres/collector.rs +++ b/crates/hir-def/src/nameres/collector.rs @@ -2466,8 +2466,8 @@ impl ModCollector<'_, '_> { None => { let explicit_name = attrs.by_key(sym::rustc_builtin_macro).tt_values().next().and_then(|tt| { - match tt.token_trees().flat_tokens().first() { - Some(tt::TokenTree::Leaf(tt::Leaf::Ident(name))) => Some(name), + match tt.token_trees().iter().next() { + Some(tt::TtElement::Leaf(tt::Leaf::Ident(name))) => Some(name), _ => None, } }); diff --git a/crates/hir-def/src/nameres/proc_macro.rs b/crates/hir-def/src/nameres/proc_macro.rs index cd45afe57d..91d664938b 100644 --- a/crates/hir-def/src/nameres/proc_macro.rs +++ b/crates/hir-def/src/nameres/proc_macro.rs @@ -2,10 +2,11 @@ use hir_expand::name::{AsName, Name}; use intern::sym; +use itertools::Itertools; use crate::{ item_tree::Attrs, - tt::{Leaf, TokenTree, TopSubtree, TtElement}, + tt::{Leaf, TopSubtree, TtElement}, }; #[derive(Debug, PartialEq, Eq)] @@ -61,35 +62,35 @@ impl Attrs<'_> { // This fn is intended for `#[proc_macro_derive(..)]` and `#[rustc_builtin_macro(..)]`, which have // the same structure. -#[rustfmt::skip] pub(crate) fn parse_macro_name_and_helper_attrs(tt: &TopSubtree) -> Option<(Name, Box<[Name]>)> { - match tt.token_trees().flat_tokens() { + if let Some([TtElement::Leaf(Leaf::Ident(trait_name))]) = + tt.token_trees().iter().collect_array() + { // `#[proc_macro_derive(Trait)]` // `#[rustc_builtin_macro(Trait)]` - [TokenTree::Leaf(Leaf::Ident(trait_name))] => Some((trait_name.as_name(), Box::new([]))), - + Some((trait_name.as_name(), Box::new([]))) + } else if let Some( + [ + TtElement::Leaf(Leaf::Ident(trait_name)), + TtElement::Leaf(Leaf::Punct(comma)), + TtElement::Leaf(Leaf::Ident(attributes)), + TtElement::Subtree(_, helpers), + ], + ) = tt.token_trees().iter().collect_array() + && comma.char == ',' + && attributes.sym == sym::attributes + { // `#[proc_macro_derive(Trait, attributes(helper1, helper2, ...))]` // `#[rustc_builtin_macro(Trait, attributes(helper1, helper2, ...))]` - [ - TokenTree::Leaf(Leaf::Ident(trait_name)), - TokenTree::Leaf(Leaf::Punct(comma)), - TokenTree::Leaf(Leaf::Ident(attributes)), - TokenTree::Subtree(_), - .. - ] if comma.char == ',' && attributes.sym == sym::attributes => - { - let helpers = tt::TokenTreesView::new(&tt.token_trees().flat_tokens()[3..]).try_into_subtree()?; - let helpers = helpers - .iter() - .filter_map(|tt| match tt { - TtElement::Leaf(Leaf::Ident(helper)) => Some(helper.as_name()), - _ => None, - }) - .collect::<Box<[_]>>(); - - Some((trait_name.as_name(), helpers)) - } + let helpers = helpers + .filter_map(|tt| match tt { + TtElement::Leaf(Leaf::Ident(helper)) => Some(helper.as_name()), + _ => None, + }) + .collect::<Box<[_]>>(); - _ => None, + Some((trait_name.as_name(), helpers)) + } else { + None } } diff --git a/crates/hir-expand/src/attrs.rs b/crates/hir-expand/src/attrs.rs index e1807cd2e1..5a44018720 100644 --- a/crates/hir-expand/src/attrs.rs +++ b/crates/hir-expand/src/attrs.rs @@ -36,6 +36,7 @@ use base_db::Crate; use cfg::{CfgExpr, CfgOptions}; use either::Either; use intern::{Interned, Symbol}; +use itertools::Itertools; use mbe::{DelimiterKind, Punct}; use parser::T; use smallvec::SmallVec; @@ -453,10 +454,10 @@ impl Attr { } /// #[path(ident)] - pub fn single_ident_value(&self) -> Option<&tt::Ident> { + pub fn single_ident_value(&self) -> Option<tt::Ident> { match self.input.as_deref()? { - AttrInput::TokenTree(tt) => match tt.token_trees().flat_tokens() { - [tt::TokenTree::Leaf(tt::Leaf::Ident(ident))] => Some(ident), + AttrInput::TokenTree(tt) => match tt.token_trees().iter().collect_array() { + Some([tt::TtElement::Leaf(tt::Leaf::Ident(ident))]) => Some(ident), _ => None, }, _ => None, @@ -492,7 +493,7 @@ fn parse_path_comma_token_tree<'a>( args.token_trees() .split(|tt| matches!(tt, tt::TtElement::Leaf(tt::Leaf::Punct(Punct { char: ',', .. })))) .filter_map(move |tts| { - let span = tts.flat_tokens().first()?.first_span(); + let span = tts.first_span()?; Some((ModPath::from_tt(db, tts)?, span, tts)) }) } @@ -611,16 +612,12 @@ impl AttrId { else { return derive_attr_range; }; - let (Some(first_tt), Some(last_tt)) = - (derive_tts.flat_tokens().first(), derive_tts.flat_tokens().last()) + let (Some(first_span), Some(last_span)) = (derive_tts.first_span(), derive_tts.last_span()) else { return derive_attr_range; }; - let start = first_tt.first_span().range.start(); - let end = match last_tt { - tt::TokenTree::Leaf(it) => it.span().range.end(), - tt::TokenTree::Subtree(it) => it.delimiter.close.range.end(), - }; + let start = first_span.range.start(); + let end = last_span.range.end(); TextRange::new(start, end) } } diff --git a/crates/hir-expand/src/builtin/fn_macro.rs b/crates/hir-expand/src/builtin/fn_macro.rs index 3d630cfc1c..011a9530c4 100644 --- a/crates/hir-expand/src/builtin/fn_macro.rs +++ b/crates/hir-expand/src/builtin/fn_macro.rs @@ -210,7 +210,7 @@ fn stringify_expand( tt: &tt::TopSubtree, span: Span, ) -> ExpandResult<tt::TopSubtree> { - let pretty = ::tt::pretty(tt.token_trees().flat_tokens()); + let pretty = ::tt::pretty(tt.token_trees()); let expanded = quote! {span => #pretty @@ -283,7 +283,7 @@ fn format_args_expand( ) -> ExpandResult<tt::TopSubtree> { let pound = mk_pound(span); let mut tt = tt.clone(); - tt.top_subtree_delimiter_mut().kind = tt::DelimiterKind::Parenthesis; + tt.set_top_subtree_delimiter_kind(tt::DelimiterKind::Parenthesis); ExpandResult::ok(quote! {span => builtin #pound format_args #tt }) @@ -297,14 +297,14 @@ fn format_args_nl_expand( ) -> ExpandResult<tt::TopSubtree> { let pound = mk_pound(span); let mut tt = tt.clone(); - tt.top_subtree_delimiter_mut().kind = tt::DelimiterKind::Parenthesis; - if let Some(tt::TokenTree::Leaf(tt::Leaf::Literal(tt::Literal { - symbol: text, - kind: tt::LitKind::Str, - .. - }))) = tt.0.get_mut(1) + tt.set_top_subtree_delimiter_kind(tt::DelimiterKind::Parenthesis); + let lit = tt.as_token_trees().iter_flat_tokens().nth(1); + if let Some(tt::TokenTree::Leaf(tt::Leaf::Literal( + mut lit @ tt::Literal { kind: tt::LitKind::Str, .. }, + ))) = lit { - *text = Symbol::intern(&format_smolstr!("{}\\n", text.as_str())); + lit.symbol = Symbol::intern(&format_smolstr!("{}\\n", lit.symbol.as_str())); + tt.set_token(1, lit.into()); } ExpandResult::ok(quote! {span => builtin #pound format_args #tt @@ -318,7 +318,7 @@ fn asm_expand( span: Span, ) -> ExpandResult<tt::TopSubtree> { let mut tt = tt.clone(); - tt.top_subtree_delimiter_mut().kind = tt::DelimiterKind::Parenthesis; + tt.set_top_subtree_delimiter_kind(tt::DelimiterKind::Parenthesis); let pound = mk_pound(span); let expanded = quote! {span => builtin #pound asm #tt @@ -333,7 +333,7 @@ fn global_asm_expand( span: Span, ) -> ExpandResult<tt::TopSubtree> { let mut tt = tt.clone(); - tt.top_subtree_delimiter_mut().kind = tt::DelimiterKind::Parenthesis; + tt.set_top_subtree_delimiter_kind(tt::DelimiterKind::Parenthesis); let pound = mk_pound(span); let expanded = quote! {span => builtin #pound global_asm #tt @@ -348,7 +348,7 @@ fn naked_asm_expand( span: Span, ) -> ExpandResult<tt::TopSubtree> { let mut tt = tt.clone(); - tt.top_subtree_delimiter_mut().kind = tt::DelimiterKind::Parenthesis; + tt.set_top_subtree_delimiter_kind(tt::DelimiterKind::Parenthesis); let pound = mk_pound(span); let expanded = quote! {span => builtin #pound naked_asm #tt @@ -478,11 +478,11 @@ fn unreachable_expand( // Pass the original arguments let mut subtree = tt.clone(); - *subtree.top_subtree_delimiter_mut() = tt::Delimiter { + subtree.set_top_subtree_delimiter_kind(tt::DelimiterKind::Parenthesis); + subtree.set_top_subtree_delimiter_span(tt::DelimSpan { open: call_site_span, close: call_site_span, - kind: tt::DelimiterKind::Parenthesis, - }; + }); // Expand to a macro call `$crate::panic::panic_{edition}` let call = quote!(call_site_span =>#dollar_crate::panic::#mac! #subtree); @@ -518,16 +518,17 @@ fn compile_error_expand( tt: &tt::TopSubtree, span: Span, ) -> ExpandResult<tt::TopSubtree> { - let err = match &*tt.0 { - [ - _, - tt::TokenTree::Leaf(tt::Leaf::Literal(tt::Literal { - symbol: text, - span: _, - kind: tt::LitKind::Str | tt::LitKind::StrRaw(_), - suffix: _, - })), - ] => ExpandError::other(span, Box::from(unescape_symbol(text).as_str())), + let err = match tt.iter().collect_array() { + Some( + [ + tt::TtElement::Leaf(tt::Leaf::Literal(tt::Literal { + symbol: text, + span: _, + kind: tt::LitKind::Str | tt::LitKind::StrRaw(_), + suffix: _, + })), + ], + ) => ExpandError::other(span, Box::from(unescape_symbol(&text).as_str())), _ => ExpandError::other(span, "`compile_error!` argument must be a string"), }; @@ -556,7 +557,7 @@ fn concat_expand( // to ensure the right parsing order, so skip the parentheses here. Ideally we'd // implement rustc's model. cc https://github.com/rust-lang/rust-analyzer/pull/10623 if let TtElement::Subtree(subtree, subtree_iter) = &t - && let [tt::TokenTree::Leaf(tt)] = subtree_iter.remaining().flat_tokens() + && let Some([tt::TtElement::Leaf(tt)]) = subtree_iter.clone().collect_array() && subtree.delimiter.kind == tt::DelimiterKind::Parenthesis { t = TtElement::Leaf(tt); @@ -663,7 +664,7 @@ fn concat_bytes_expand( kind, suffix: _, })) => { - record_span(*span); + record_span(span); match kind { tt::LitKind::Byte => { if let Ok(b) = unescape_byte(text.as_str()) { @@ -679,7 +680,7 @@ fn concat_bytes_expand( bytes.extend(text.as_str().escape_debug()); } _ => { - err.get_or_insert(ExpandError::other(*span, "unexpected token")); + err.get_or_insert(ExpandError::other(span, "unexpected token")); break; } } @@ -733,7 +734,7 @@ fn concat_bytes_expand_subtree( if let Ok(b) = unescape_byte(text.as_str()) { bytes.extend(b.escape_ascii().filter_map(|it| char::from_u32(it as u32))); } - record_span(*span); + record_span(span); } TtElement::Leaf(tt::Leaf::Literal(tt::Literal { symbol: text, @@ -741,7 +742,7 @@ fn concat_bytes_expand_subtree( kind: tt::LitKind::Integer, suffix: _, })) => { - record_span(*span); + record_span(span); if let Ok(b) = text.as_str().parse::<u8>() { bytes.extend(b.escape_ascii().filter_map(|it| char::from_u32(it as u32))); } @@ -796,13 +797,13 @@ fn parse_string(tt: &tt::TopSubtree) -> Result<(Symbol, Span), ExpandError> { span, kind: tt::LitKind::Str, suffix: _, - })) => Ok((unescape_symbol(text), *span)), + })) => Ok((unescape_symbol(&text), span)), TtElement::Leaf(tt::Leaf::Literal(tt::Literal { symbol: text, span, kind: tt::LitKind::StrRaw(_), suffix: _, - })) => Ok((text.clone(), *span)), + })) => Ok((text.clone(), span)), TtElement::Leaf(l) => Err(*l.span()), TtElement::Subtree(tt, _) => Err(tt.delimiter.open.cover(tt.delimiter.close)), } diff --git a/crates/hir-expand/src/builtin/quote.rs b/crates/hir-expand/src/builtin/quote.rs index b1d46f5c4e..4039b6c334 100644 --- a/crates/hir-expand/src/builtin/quote.rs +++ b/crates/hir-expand/src/builtin/quote.rs @@ -163,7 +163,7 @@ impl ToTokenTree for crate::tt::SubtreeView<'_> { impl ToTokenTree for crate::tt::TopSubtree { fn to_tokens(self, _: Span, builder: &mut TopSubtreeBuilder) { - builder.extend_tt_dangerous(self.0); + builder.extend_with_tt(self.as_token_trees()); } } @@ -172,10 +172,9 @@ impl ToTokenTree for crate::tt::TtElement<'_> { match self { crate::tt::TtElement::Leaf(leaf) => builder.push(leaf.clone()), crate::tt::TtElement::Subtree(subtree, subtree_iter) => { - builder.extend_tt_dangerous( - std::iter::once(crate::tt::TokenTree::Subtree(subtree.clone())) - .chain(subtree_iter.remaining().flat_tokens().iter().cloned()), - ); + builder.open(subtree.delimiter.kind, subtree.delimiter.open); + builder.extend_with_tt(subtree_iter.remaining()); + builder.close(subtree.delimiter.close); } } } diff --git a/crates/hir-expand/src/db.rs b/crates/hir-expand/src/db.rs index 5c517e671b..40d44cd1db 100644 --- a/crates/hir-expand/src/db.rs +++ b/crates/hir-expand/src/db.rs @@ -237,7 +237,8 @@ pub fn expand_speculative( span, DocCommentDesugarMode::ProcMacro, ); - *tree.top_subtree_delimiter_mut() = tt::Delimiter::invisible_spanned(span); + tree.set_top_subtree_delimiter_kind(tt::DelimiterKind::Invisible); + tree.set_top_subtree_delimiter_span(tt::DelimSpan::from_single(span)); tree }, ) @@ -255,7 +256,7 @@ pub fn expand_speculative( span, DocCommentDesugarMode::ProcMacro, ); - attr_arg.top_subtree_delimiter_mut().kind = tt::DelimiterKind::Invisible; + attr_arg.set_top_subtree_delimiter_kind(tt::DelimiterKind::Invisible); Some(attr_arg) } _ => None, @@ -270,7 +271,8 @@ pub fn expand_speculative( let mut speculative_expansion = match loc.def.kind { MacroDefKind::ProcMacro(ast, expander, _) => { let span = db.proc_macro_span(ast); - *tt.top_subtree_delimiter_mut() = tt::Delimiter::invisible_spanned(span); + tt.set_top_subtree_delimiter_kind(tt::DelimiterKind::Invisible); + tt.set_top_subtree_delimiter_span(tt::DelimSpan::from_single(span)); expander.expand( db, loc.def.krate, @@ -430,7 +432,7 @@ fn macro_arg(db: &dyn ExpandDatabase, id: MacroCallId) -> MacroArgResult { ( Arc::new(tt::TopSubtree::from_token_trees( tt::Delimiter { open: span, close: span, kind }, - tt::TokenTreesView::new(&[]), + tt::TokenTreesView::empty(), )), SyntaxFixupUndoInfo::default(), span, @@ -478,7 +480,7 @@ fn macro_arg(db: &dyn ExpandDatabase, id: MacroCallId) -> MacroArgResult { ); if loc.def.is_proc_macro() { // proc macros expect their inputs without parentheses, MBEs expect it with them included - tt.top_subtree_delimiter_mut().kind = tt::DelimiterKind::Invisible; + tt.set_top_subtree_delimiter_kind(tt::DelimiterKind::Invisible); } return (Arc::new(tt), SyntaxFixupUndoInfo::NONE, span); } @@ -512,7 +514,7 @@ fn macro_arg(db: &dyn ExpandDatabase, id: MacroCallId) -> MacroArgResult { if loc.def.is_proc_macro() { // proc macros expect their inputs without parentheses, MBEs expect it with them included - tt.top_subtree_delimiter_mut().kind = tt::DelimiterKind::Invisible; + tt.set_top_subtree_delimiter_kind(tt::DelimiterKind::Invisible); } (Arc::new(tt), undo_info, span) diff --git a/crates/hir-expand/src/eager.rs b/crates/hir-expand/src/eager.rs index 9b65bdac65..0b6124ebf3 100644 --- a/crates/hir-expand/src/eager.rs +++ b/crates/hir-expand/src/eager.rs @@ -96,7 +96,7 @@ pub fn expand_eager_macro_input( DocCommentDesugarMode::Mbe, ); - subtree.top_subtree_delimiter_mut().kind = crate::tt::DelimiterKind::Invisible; + subtree.set_top_subtree_delimiter_kind(crate::tt::DelimiterKind::Invisible); let loc = MacroCallLoc { def, diff --git a/crates/hir-expand/src/fixup.rs b/crates/hir-expand/src/fixup.rs index 876d870936..6afc6e5d5e 100644 --- a/crates/hir-expand/src/fixup.rs +++ b/crates/hir-expand/src/fixup.rs @@ -15,7 +15,7 @@ use syntax::{ }; use syntax_bridge::DocCommentDesugarMode; use triomphe::Arc; -use tt::Spacing; +use tt::{Spacing, TransformTtAction, transform_tt}; use crate::{ span_map::SpanMapRef, @@ -343,93 +343,29 @@ fn has_error_to_handle(node: &SyntaxNode) -> bool { pub(crate) fn reverse_fixups(tt: &mut TopSubtree, undo_info: &SyntaxFixupUndoInfo) { let Some(undo_info) = undo_info.original.as_deref() else { return }; let undo_info = &**undo_info; - let delimiter = tt.top_subtree_delimiter_mut(); + let top_subtree = tt.top_subtree(); + let open_span = top_subtree.delimiter.open; + let close_span = top_subtree.delimiter.close; #[allow(deprecated)] if never!( - delimiter.close.anchor.ast_id == FIXUP_DUMMY_AST_ID - || delimiter.open.anchor.ast_id == FIXUP_DUMMY_AST_ID + close_span.anchor.ast_id == FIXUP_DUMMY_AST_ID + || open_span.anchor.ast_id == FIXUP_DUMMY_AST_ID ) { let span = |file_id| Span { range: TextRange::empty(TextSize::new(0)), anchor: SpanAnchor { file_id, ast_id: ROOT_ERASED_FILE_AST_ID }, ctx: SyntaxContext::root(span::Edition::Edition2015), }; - delimiter.open = span(delimiter.open.anchor.file_id); - delimiter.close = span(delimiter.close.anchor.file_id); + tt.set_top_subtree_delimiter_span(tt::DelimSpan { + open: span(open_span.anchor.file_id), + close: span(close_span.anchor.file_id), + }); } reverse_fixups_(tt, undo_info); } -#[derive(Debug)] -enum TransformTtAction<'a> { - Keep, - ReplaceWith(tt::TokenTreesView<'a>), -} - -impl TransformTtAction<'_> { - fn remove() -> Self { - Self::ReplaceWith(tt::TokenTreesView::new(&[])) - } -} - -/// This function takes a token tree, and calls `callback` with each token tree in it. -/// Then it does what the callback says: keeps the tt or replaces it with a (possibly empty) -/// tts view. -fn transform_tt<'a, 'b>( - tt: &'a mut Vec<tt::TokenTree>, - mut callback: impl FnMut(&mut tt::TokenTree) -> TransformTtAction<'b>, -) { - // We need to keep a stack of the currently open subtrees, because we need to update - // them if we change the number of items in them. - let mut subtrees_stack = Vec::new(); - let mut i = 0; - while i < tt.len() { - 'pop_finished_subtrees: while let Some(&subtree_idx) = subtrees_stack.last() { - let tt::TokenTree::Subtree(subtree) = &tt[subtree_idx] else { - unreachable!("non-subtree on subtrees stack"); - }; - if i >= subtree_idx + 1 + subtree.usize_len() { - subtrees_stack.pop(); - } else { - break 'pop_finished_subtrees; - } - } - - let action = callback(&mut tt[i]); - match action { - TransformTtAction::Keep => { - // This cannot be shared with the replaced case, because then we may push the same subtree - // twice, and will update it twice which will lead to errors. - if let tt::TokenTree::Subtree(_) = &tt[i] { - subtrees_stack.push(i); - } - - i += 1; - } - TransformTtAction::ReplaceWith(replacement) => { - let old_len = 1 + match &tt[i] { - tt::TokenTree::Leaf(_) => 0, - tt::TokenTree::Subtree(subtree) => subtree.usize_len(), - }; - let len_diff = replacement.len() as i64 - old_len as i64; - tt.splice(i..i + old_len, replacement.flat_tokens().iter().cloned()); - // Skip the newly inserted replacement, we don't want to visit it. - i += replacement.len(); - - for &subtree_idx in &subtrees_stack { - let tt::TokenTree::Subtree(subtree) = &mut tt[subtree_idx] else { - unreachable!("non-subtree on subtrees stack"); - }; - subtree.len = (i64::from(subtree.len) + len_diff).try_into().unwrap(); - } - } - } - } -} - fn reverse_fixups_(tt: &mut TopSubtree, undo_info: &[TopSubtree]) { - let mut tts = std::mem::take(&mut tt.0).into_vec(); - transform_tt(&mut tts, |tt| match tt { + transform_tt(tt, |tt| match tt { tt::TokenTree::Leaf(leaf) => { let span = leaf.span(); let is_real_leaf = span.anchor.ast_id != FIXUP_DUMMY_AST_ID; @@ -459,7 +395,6 @@ fn reverse_fixups_(tt: &mut TopSubtree, undo_info: &[TopSubtree]) { TransformTtAction::Keep } }); - tt.0 = tts.into_boxed_slice(); } #[cfg(test)] @@ -488,9 +423,9 @@ mod tests { } fn check_subtree_eq(a: &tt::TopSubtree, b: &tt::TopSubtree) -> bool { - let a = a.view().as_token_trees().flat_tokens(); - let b = b.view().as_token_trees().flat_tokens(); - a.len() == b.len() && std::iter::zip(a, b).all(|(a, b)| check_tt_eq(a, b)) + let a = a.view().as_token_trees().iter_flat_tokens(); + let b = b.view().as_token_trees().iter_flat_tokens(); + a.len() == b.len() && std::iter::zip(a, b).all(|(a, b)| check_tt_eq(&a, &b)) } fn check_tt_eq(a: &tt::TokenTree, b: &tt::TokenTree) -> bool { @@ -545,7 +480,7 @@ mod tests { // the fixed-up tree should not contain braces as punct // FIXME: should probably instead check that it's a valid punctuation character - for x in tt.token_trees().flat_tokens() { + for x in tt.token_trees().iter_flat_tokens() { match x { ::tt::TokenTree::Leaf(::tt::Leaf::Punct(punct)) => { assert!(!matches!(punct.char, '{' | '}' | '(' | ')' | '[' | ']')) diff --git a/crates/hir-expand/src/mod_path.rs b/crates/hir-expand/src/mod_path.rs index e9805e3f86..51e449fe50 100644 --- a/crates/hir-expand/src/mod_path.rs +++ b/crates/hir-expand/src/mod_path.rs @@ -355,16 +355,16 @@ fn convert_path_tt(db: &dyn ExpandDatabase, tt: tt::TokenTreesView<'_>) -> Optio tt::Leaf::Punct(tt::Punct { char: ':', .. }) => PathKind::Abs, _ => return None, }, - tt::Leaf::Ident(tt::Ident { sym: text, span, .. }) if *text == sym::dollar_crate => { + tt::Leaf::Ident(tt::Ident { sym: text, span, .. }) if text == sym::dollar_crate => { resolve_crate_root(db, span.ctx).map(PathKind::DollarCrate).unwrap_or(PathKind::Crate) } - tt::Leaf::Ident(tt::Ident { sym: text, .. }) if *text == sym::self_ => PathKind::SELF, - tt::Leaf::Ident(tt::Ident { sym: text, .. }) if *text == sym::super_ => { + tt::Leaf::Ident(tt::Ident { sym: text, .. }) if text == sym::self_ => PathKind::SELF, + tt::Leaf::Ident(tt::Ident { sym: text, .. }) if text == sym::super_ => { let mut deg = 1; while let Some(tt::Leaf::Ident(tt::Ident { sym: text, span, is_raw: _ })) = leaves.next() { - if *text != sym::super_ { + if text != sym::super_ { segments.push(Name::new_symbol(text.clone(), span.ctx)); break; } @@ -372,7 +372,7 @@ fn convert_path_tt(db: &dyn ExpandDatabase, tt: tt::TokenTreesView<'_>) -> Optio } PathKind::Super(deg) } - tt::Leaf::Ident(tt::Ident { sym: text, .. }) if *text == sym::crate_ => PathKind::Crate, + tt::Leaf::Ident(tt::Ident { sym: text, .. }) if text == sym::crate_ => PathKind::Crate, tt::Leaf::Ident(ident) => { segments.push(Name::new_symbol(ident.sym.clone(), ident.span.ctx)); PathKind::Plain diff --git a/crates/mbe/src/benchmark.rs b/crates/mbe/src/benchmark.rs index e0f1e616d8..6b018510be 100644 --- a/crates/mbe/src/benchmark.rs +++ b/crates/mbe/src/benchmark.rs @@ -54,7 +54,7 @@ fn benchmark_expand_macro_rules() { .map(|(id, tt)| { let res = rules[&id].expand(&db, &tt, |_| (), MacroCallStyle::FnLike, DUMMY); assert!(res.err.is_none()); - res.value.0.0.len() + res.value.0.as_token_trees().len() }) .sum() }; diff --git a/crates/mbe/src/expander.rs b/crates/mbe/src/expander.rs index 36c7871519..25e05df7b7 100644 --- a/crates/mbe/src/expander.rs +++ b/crates/mbe/src/expander.rs @@ -162,7 +162,7 @@ impl Fragment<'_> { Fragment::Tokens { tree, .. } => tree.len() == 0, Fragment::Expr(it) => it.len() == 0, Fragment::Path(it) => it.len() == 0, - Fragment::TokensOwned(it) => it.0.is_empty(), + Fragment::TokensOwned(_) => false, // A `TopSubtree` is never empty } } } diff --git a/crates/mbe/src/expander/matcher.rs b/crates/mbe/src/expander/matcher.rs index c4c1e1f565..e845b1ab8d 100644 --- a/crates/mbe/src/expander/matcher.rs +++ b/crates/mbe/src/expander/matcher.rs @@ -517,7 +517,7 @@ fn match_loop_inner<'t>( } OpDelimited::Op(Op::Literal(lhs)) => { if let Ok(rhs) = src.clone().expect_leaf() { - if matches!(rhs, tt::Leaf::Literal(it) if it.symbol == lhs.symbol) { + if matches!(&rhs, tt::Leaf::Literal(it) if it.symbol == lhs.symbol) { item.dot.next(); } else { res.add_err(ExpandError::new( @@ -537,7 +537,7 @@ fn match_loop_inner<'t>( } OpDelimited::Op(Op::Ident(lhs)) => { if let Ok(rhs) = src.clone().expect_leaf() { - if matches!(rhs, tt::Leaf::Ident(it) if it.sym == lhs.sym) { + if matches!(&rhs, tt::Leaf::Ident(it) if it.sym == lhs.sym) { item.dot.next(); } else { res.add_err(ExpandError::new( @@ -701,7 +701,7 @@ fn match_loop<'t>( || !(bb_items.is_empty() || next_items.is_empty()) || bb_items.len() > 1; if has_leftover_tokens { - res.unmatched_tts += src.remaining().flat_tokens().len(); + res.unmatched_tts += src.remaining().len(); res.add_err(ExpandError::new(span.open, ExpandErrorKind::LeftoverTokens)); if let Some(error_recover_item) = error_recover_item { @@ -991,7 +991,7 @@ fn expect_tt(iter: &mut TtIter<'_>) -> Result<(), ()> { Ok(()) } -fn expect_lifetime<'a>(iter: &mut TtIter<'a>) -> Result<&'a tt::Ident, ()> { +fn expect_lifetime<'a>(iter: &mut TtIter<'a>) -> Result<tt::Ident, ()> { let punct = iter.expect_single_punct()?; if punct.char != '\'' { return Err(()); @@ -1000,7 +1000,7 @@ fn expect_lifetime<'a>(iter: &mut TtIter<'a>) -> Result<&'a tt::Ident, ()> { } fn eat_char(iter: &mut TtIter<'_>, c: char) { - if matches!(iter.peek(), Some(TtElement::Leaf(tt::Leaf::Punct(tt::Punct { char, .. }))) if *char == c) + if matches!(iter.peek(), Some(TtElement::Leaf(tt::Leaf::Punct(tt::Punct { char, .. }))) if char == c) { iter.next().expect("already peeked"); } diff --git a/crates/mbe/src/expander/transcriber.rs b/crates/mbe/src/expander/transcriber.rs index a0fd4550ba..38098e2c84 100644 --- a/crates/mbe/src/expander/transcriber.rs +++ b/crates/mbe/src/expander/transcriber.rs @@ -3,6 +3,7 @@ use intern::{Symbol, sym}; use span::{Edition, Span}; +use stdx::itertools::Itertools; use tt::{Delimiter, TopSubtreeBuilder, iter::TtElement}; use super::TokensOrigin; @@ -324,7 +325,7 @@ fn expand_subtree( } _ => (None, None), }; - let value = match values { + let value = match &values { (Some(TtElement::Leaf(tt::Leaf::Ident(ident))), None) => { ident.sym.as_str() } @@ -412,15 +413,15 @@ fn expand_var( // Check if this is a simple negative literal (MINUS + LITERAL) // that should not be wrapped in parentheses let is_negative_literal = matches!( - sub.flat_tokens(), - [ - tt::TokenTree::Leaf(tt::Leaf::Punct(tt::Punct { char: '-', .. })), - tt::TokenTree::Leaf(tt::Leaf::Literal(_)) - ] + sub.iter().collect_array(), + Some([ + tt::TtElement::Leaf(tt::Leaf::Punct(tt::Punct { char: '-', .. })), + tt::TtElement::Leaf(tt::Leaf::Literal(_)) + ]) ); let wrap_in_parens = !is_negative_literal - && !matches!(sub.flat_tokens(), [tt::TokenTree::Leaf(_)]) + && !matches!(sub.iter().collect_array(), Some([tt::TtElement::Leaf(_)])) && sub.try_into_subtree().is_none_or(|it| { it.top_subtree().delimiter.kind == tt::DelimiterKind::Invisible }); @@ -560,8 +561,8 @@ fn fix_up_and_push_path_tt( // argument list and thus needs `::` between it and `FnOnce`. However in // today's Rust this type of path *semantically* cannot appear as a // top-level expression-context path, so we can safely ignore it. - if let [tt::TokenTree::Leaf(tt::Leaf::Punct(tt::Punct { char: '<', .. }))] = - tt.flat_tokens() + if let Some([tt::TtElement::Leaf(tt::Leaf::Punct(tt::Punct { char: '<', .. }))]) = + tt.iter().collect_array() { builder.extend([ tt::Leaf::Punct(tt::Punct { @@ -577,7 +578,8 @@ fn fix_up_and_push_path_tt( ]); } } - prev_was_ident = matches!(tt.flat_tokens(), [tt::TokenTree::Leaf(tt::Leaf::Ident(_))]); + prev_was_ident = + matches!(tt.iter().collect_array(), Some([tt::TtElement::Leaf(tt::Leaf::Ident(_))])); builder.extend_with_tt(tt); } } diff --git a/crates/mbe/src/parser.rs b/crates/mbe/src/parser.rs index ddf9afbf98..cecdd43a42 100644 --- a/crates/mbe/src/parser.rs +++ b/crates/mbe/src/parser.rs @@ -224,7 +224,7 @@ fn next_op( None => { return Ok(Op::Punct({ let mut res = ArrayVec::new(); - res.push(*p); + res.push(p); Box::new(res) })); } @@ -268,7 +268,7 @@ fn next_op( let id = ident.span; Op::Var { name, kind, id } } - tt::Leaf::Literal(lit) if is_boolean_literal(lit) => { + tt::Leaf::Literal(lit) if is_boolean_literal(&lit) => { let kind = eat_fragment_kind(edition, src, mode)?; let name = lit.symbol.clone(); let id = lit.span; @@ -282,7 +282,7 @@ fn next_op( } Mode::Template => Op::Punct({ let mut res = ArrayVec::new(); - res.push(*punct); + res.push(punct); Box::new(res) }), }, @@ -400,7 +400,7 @@ fn parse_repeat(src: &mut TtIter<'_>) -> Result<(Option<Separator>, RepeatKind), '?' => RepeatKind::ZeroOrOne, _ => match &mut separator { Separator::Puncts(puncts) if puncts.len() < 3 => { - puncts.push(*punct); + puncts.push(punct); continue; } _ => return Err(ParseError::InvalidRepeat), diff --git a/crates/proc-macro-api/src/legacy_protocol/msg/flat.rs b/crates/proc-macro-api/src/legacy_protocol/msg/flat.rs index 8c96b06700..ed4b7aff76 100644 --- a/crates/proc-macro-api/src/legacy_protocol/msg/flat.rs +++ b/crates/proc-macro-api/src/legacy_protocol/msg/flat.rs @@ -489,7 +489,7 @@ struct Writer<'a, 'span, S: SpanTransformer, W> { impl<'a, T: SpanTransformer<Span = span::Span>> Writer<'a, '_, T, tt::iter::TtIter<'a>> { fn write_subtree(&mut self, root: tt::SubtreeView<'a>) { let subtree = root.top_subtree(); - self.enqueue(subtree, root.iter()); + self.enqueue(&subtree, root.iter()); while let Some((idx, len, subtree)) = self.work.pop_front() { self.subtree(idx, len, subtree); } @@ -504,7 +504,7 @@ impl<'a, T: SpanTransformer<Span = span::Span>> Writer<'a, '_, T, tt::iter::TtIt for child in subtree { let idx_tag = match child { tt::iter::TtElement::Subtree(subtree, subtree_iter) => { - let idx = self.enqueue(subtree, subtree_iter); + let idx = self.enqueue(&subtree, subtree_iter); idx << 2 } tt::iter::TtElement::Leaf(leaf) => match leaf { @@ -513,8 +513,11 @@ impl<'a, T: SpanTransformer<Span = span::Span>> Writer<'a, '_, T, tt::iter::TtIt let id = self.token_id_of(lit.span); let (text, suffix) = if self.version >= EXTENDED_LEAF_DATA { ( - self.intern(lit.symbol.as_str()), - lit.suffix.as_ref().map(|s| self.intern(s.as_str())).unwrap_or(!0), + self.intern_owned(lit.symbol.as_str().to_owned()), + lit.suffix + .as_ref() + .map(|s| self.intern_owned(s.as_str().to_owned())) + .unwrap_or(!0), ) } else { (self.intern_owned(format!("{lit}")), !0) @@ -549,11 +552,11 @@ impl<'a, T: SpanTransformer<Span = span::Span>> Writer<'a, '_, T, tt::iter::TtIt let idx = self.ident.len() as u32; let id = self.token_id_of(ident.span); let text = if self.version >= EXTENDED_LEAF_DATA { - self.intern(ident.sym.as_str()) + self.intern_owned(ident.sym.as_str().to_owned()) } else if ident.is_raw.yes() { self.intern_owned(format!("r#{}", ident.sym.as_str(),)) } else { - self.intern(ident.sym.as_str()) + self.intern_owned(ident.sym.as_str().to_owned()) }; self.ident.push(IdentRepr { id, text, is_raw: ident.is_raw.yes() }); (idx << 2) | 0b11 @@ -565,7 +568,7 @@ impl<'a, T: SpanTransformer<Span = span::Span>> Writer<'a, '_, T, tt::iter::TtIt } } - fn enqueue(&mut self, subtree: &'a tt::Subtree, contents: tt::iter::TtIter<'a>) -> u32 { + fn enqueue(&mut self, subtree: &tt::Subtree, contents: tt::iter::TtIter<'a>) -> u32 { let idx = self.subtree.len(); let open = self.token_id_of(subtree.delimiter.open); let close = self.token_id_of(subtree.delimiter.close); @@ -582,6 +585,7 @@ impl<'a, T: SpanTransformer, U> Writer<'a, '_, T, U> { T::token_id_of(self.span_data_table, span) } + #[cfg(feature = "sysroot-abi")] pub(crate) fn intern(&mut self, text: &'a str) -> u32 { let table = &mut self.text; *self.string_table.entry(text.into()).or_insert_with(|| { @@ -840,7 +844,7 @@ impl<T: SpanTransformer<Span = span::Span>> Reader<'_, T> { let (delimiter, mut res) = res[0].take().unwrap(); res.insert(0, tt::TokenTree::Subtree(tt::Subtree { delimiter, len: res.len() as u32 })); - tt::TopSubtree(res.into_boxed_slice()) + tt::TopSubtree::from_serialized(res) } } diff --git a/crates/proc-macro-api/src/lib.rs b/crates/proc-macro-api/src/lib.rs index e0d20c82b9..f5fcc99f14 100644 --- a/crates/proc-macro-api/src/lib.rs +++ b/crates/proc-macro-api/src/lib.rs @@ -194,28 +194,14 @@ impl ProcMacro { /// On some server versions, the fixup ast id is different than ours. So change it to match. fn change_fixup_to_match_old_server(&self, tt: &mut tt::TopSubtree) { const OLD_FIXUP_AST_ID: ErasedFileAstId = ErasedFileAstId::from_raw(!0 - 1); - let change_ast_id = |ast_id: &mut ErasedFileAstId| { + tt.change_every_ast_id(|ast_id| { if *ast_id == FIXUP_ERASED_FILE_AST_ID_MARKER { *ast_id = OLD_FIXUP_AST_ID; } else if *ast_id == OLD_FIXUP_AST_ID { // Swap between them, that means no collision plus the change can be reversed by doing itself. *ast_id = FIXUP_ERASED_FILE_AST_ID_MARKER; } - }; - - for tt in &mut tt.0 { - match tt { - tt::TokenTree::Leaf(tt::Leaf::Ident(tt::Ident { span, .. })) - | tt::TokenTree::Leaf(tt::Leaf::Literal(tt::Literal { span, .. })) - | tt::TokenTree::Leaf(tt::Leaf::Punct(tt::Punct { span, .. })) => { - change_ast_id(&mut span.anchor.ast_id); - } - tt::TokenTree::Subtree(subtree) => { - change_ast_id(&mut subtree.delimiter.open.anchor.ast_id); - change_ast_id(&mut subtree.delimiter.close.anchor.ast_id); - } - } - } + }); } /// Expands the procedural macro by sending an expansion request to the server. diff --git a/crates/syntax-bridge/src/lib.rs b/crates/syntax-bridge/src/lib.rs index 82cf51f162..79b51a816e 100644 --- a/crates/syntax-bridge/src/lib.rs +++ b/crates/syntax-bridge/src/lib.rs @@ -223,7 +223,7 @@ where spacing: _, })) => { let found_expected_delimiter = - builder.expected_delimiters().enumerate().find(|(_, delim)| match delim.kind { + builder.expected_delimiters().enumerate().find(|(_, delim)| match delim { tt::DelimiterKind::Parenthesis => char == ')', tt::DelimiterKind::Brace => char == '}', tt::DelimiterKind::Bracket => char == ']', @@ -257,13 +257,11 @@ where } kind if kind.is_punct() && kind != UNDERSCORE => { let found_expected_delimiter = - builder.expected_delimiters().enumerate().find(|(_, delim)| { - match delim.kind { - tt::DelimiterKind::Parenthesis => kind == T![')'], - tt::DelimiterKind::Brace => kind == T!['}'], - tt::DelimiterKind::Bracket => kind == T![']'], - tt::DelimiterKind::Invisible => false, - } + builder.expected_delimiters().enumerate().find(|(_, delim)| match delim { + tt::DelimiterKind::Parenthesis => kind == T![')'], + tt::DelimiterKind::Brace => kind == T!['}'], + tt::DelimiterKind::Bracket => kind == T![']'], + tt::DelimiterKind::Invisible => false, }); // Current token is a closing delimiter that we expect, fix up the closing span diff --git a/crates/syntax-bridge/src/tests.rs b/crates/syntax-bridge/src/tests.rs index c8dc3131b5..8c28e1c5aa 100644 --- a/crates/syntax-bridge/src/tests.rs +++ b/crates/syntax-bridge/src/tests.rs @@ -30,7 +30,7 @@ fn check_punct_spacing(fixture: &str) { }) .collect(); - let mut cursor = Cursor::new(&subtree.0); + let mut cursor = Cursor::new(subtree.as_token_trees()); while !cursor.eof() { while let Some(token_tree) = cursor.token_tree() { if let tt::TokenTree::Leaf(Leaf::Punct(Punct { diff --git a/crates/test-fixture/src/lib.rs b/crates/test-fixture/src/lib.rs index 67f69d0fa9..487171c3e3 100644 --- a/crates/test-fixture/src/lib.rs +++ b/crates/test-fixture/src/lib.rs @@ -770,7 +770,7 @@ impl ProcMacroExpander for Issue18089ProcMacroExpander { _: Span, _: String, ) -> Result<TopSubtree, ProcMacroExpansionError> { - let tt::TokenTree::Leaf(macro_name) = &subtree.0[2] else { + let Some(tt::TtElement::Leaf(macro_name)) = subtree.iter().nth(1) else { return Err(ProcMacroExpansionError::Panic("incorrect input".to_owned())); }; Ok(quote! { call_site => @@ -837,13 +837,14 @@ impl ProcMacroExpander for Issue18840ProcMacroExpander { // ``` // The span that was created by the fixup infra. - let fixed_up_span = fn_.token_trees().flat_tokens()[5].first_span(); + let mut iter = fn_.iter(); + iter.nth(2); + let (_, mut fn_body) = iter.expect_subtree().unwrap(); + let fixed_up_span = fn_body.nth(1).unwrap().first_span(); let mut result = quote! {fixed_up_span => ::core::compile_error! { "my cool compile_error!" } }; // Make it so we won't remove the top subtree when reversing fixups. - let top_subtree_delimiter_mut = result.top_subtree_delimiter_mut(); - top_subtree_delimiter_mut.open = def_site; - top_subtree_delimiter_mut.close = def_site; + result.set_top_subtree_delimiter_span(tt::DelimSpan::from_single(def_site)); Ok(result) } @@ -905,13 +906,14 @@ impl ProcMacroExpander for ShortenProcMacroExpander { _: Span, _: String, ) -> Result<TopSubtree, ProcMacroExpansionError> { - let mut result = input.0.clone(); - for it in &mut result { - if let TokenTree::Leaf(leaf) = it { - modify_leaf(leaf) + let mut result = input.clone(); + for (idx, it) in input.as_token_trees().iter_flat_tokens().enumerate() { + if let TokenTree::Leaf(mut leaf) = it { + modify_leaf(&mut leaf); + result.set_token(idx, leaf); } } - return Ok(tt::TopSubtree(result)); + return Ok(result); fn modify_leaf(leaf: &mut Leaf) { match leaf { @@ -948,7 +950,8 @@ impl ProcMacroExpander for Issue17479ProcMacroExpander { _: Span, _: String, ) -> Result<TopSubtree, ProcMacroExpansionError> { - let TokenTree::Leaf(Leaf::Literal(lit)) = &subtree.0[1] else { + let mut iter = subtree.iter(); + let Some(TtElement::Leaf(tt::Leaf::Literal(lit))) = iter.next() else { return Err(ProcMacroExpansionError::Panic("incorrect Input".into())); }; let symbol = &lit.symbol; @@ -980,10 +983,8 @@ impl ProcMacroExpander for Issue18898ProcMacroExpander { ) -> Result<TopSubtree, ProcMacroExpansionError> { let span = subtree .token_trees() - .flat_tokens() - .last() - .ok_or_else(|| ProcMacroExpansionError::Panic("malformed input".to_owned()))? - .first_span(); + .last_span() + .ok_or_else(|| ProcMacroExpansionError::Panic("malformed input".to_owned()))?; let overly_long_subtree = quote! {span => { let a = 5; @@ -1034,7 +1035,7 @@ impl ProcMacroExpander for DisallowCfgProcMacroExpander { _: Span, _: String, ) -> Result<TopSubtree, ProcMacroExpansionError> { - for tt in subtree.token_trees().flat_tokens() { + for tt in subtree.token_trees().iter_flat_tokens() { if let tt::TokenTree::Leaf(tt::Leaf::Ident(ident)) = tt && (ident.sym == sym::cfg || ident.sym == sym::cfg_attr) { @@ -1066,20 +1067,23 @@ impl ProcMacroExpander for GenerateSuffixedTypeProcMacroExpander { _mixed_site: Span, _current_dir: String, ) -> Result<TopSubtree, ProcMacroExpansionError> { - let TokenTree::Leaf(Leaf::Ident(ident)) = &subtree.0[1] else { + let mut iter = subtree.iter(); + let Some(TtElement::Leaf(tt::Leaf::Ident(ident))) = iter.next() else { return Err(ProcMacroExpansionError::Panic("incorrect Input".into())); }; let ident = match ident.sym.as_str() { "struct" => { - let TokenTree::Leaf(Leaf::Ident(ident)) = &subtree.0[2] else { + let Some(TtElement::Leaf(tt::Leaf::Ident(ident))) = iter.next() else { return Err(ProcMacroExpansionError::Panic("incorrect Input".into())); }; ident } "enum" => { - let TokenTree::Leaf(Leaf::Ident(ident)) = &subtree.0[4] else { + iter.next(); + let (_, mut iter) = iter.expect_subtree().unwrap(); + let Some(TtElement::Leaf(tt::Leaf::Ident(ident))) = iter.next() else { return Err(ProcMacroExpansionError::Panic("incorrect Input".into())); }; ident diff --git a/crates/tt/src/buffer.rs b/crates/tt/src/buffer.rs index c464e5ece1..de6379b5cd 100644 --- a/crates/tt/src/buffer.rs +++ b/crates/tt/src/buffer.rs @@ -10,8 +10,8 @@ pub struct Cursor<'a> { } impl<'a> Cursor<'a> { - pub fn new(buffer: &'a [TokenTree]) -> Self { - Self { buffer, index: 0, subtrees_stack: Vec::new() } + pub fn new(buffer: TokenTreesView<'a>) -> Self { + Self { buffer: buffer.0, index: 0, subtrees_stack: Vec::new() } } /// Check whether it is eof diff --git a/crates/tt/src/iter.rs b/crates/tt/src/iter.rs index 88c3c7f52e..5ab9f94b63 100644 --- a/crates/tt/src/iter.rs +++ b/crates/tt/src/iter.rs @@ -36,28 +36,28 @@ impl<'a> TtIter<'a> { pub fn expect_char(&mut self, char: char) -> Result<(), ()> { match self.next() { - Some(TtElement::Leaf(&Leaf::Punct(Punct { char: c, .. }))) if c == char => Ok(()), + Some(TtElement::Leaf(Leaf::Punct(Punct { char: c, .. }))) if c == char => Ok(()), _ => Err(()), } } pub fn expect_any_char(&mut self, chars: &[char]) -> Result<(), ()> { match self.next() { - Some(TtElement::Leaf(Leaf::Punct(Punct { char: c, .. }))) if chars.contains(c) => { + Some(TtElement::Leaf(Leaf::Punct(Punct { char: c, .. }))) if chars.contains(&c) => { Ok(()) } _ => Err(()), } } - pub fn expect_subtree(&mut self) -> Result<(&'a Subtree, TtIter<'a>), ()> { + pub fn expect_subtree(&mut self) -> Result<(Subtree, TtIter<'a>), ()> { match self.next() { Some(TtElement::Subtree(subtree, iter)) => Ok((subtree, iter)), _ => Err(()), } } - pub fn expect_leaf(&mut self) -> Result<&'a Leaf, ()> { + pub fn expect_leaf(&mut self) -> Result<Leaf, ()> { match self.next() { Some(TtElement::Leaf(it)) => Ok(it), _ => Err(()), @@ -78,30 +78,30 @@ impl<'a> TtIter<'a> { } } - pub fn expect_ident(&mut self) -> Result<&'a Ident, ()> { + pub fn expect_ident(&mut self) -> Result<Ident, ()> { match self.expect_leaf()? { Leaf::Ident(it) if it.sym != sym::underscore => Ok(it), _ => Err(()), } } - pub fn expect_ident_or_underscore(&mut self) -> Result<&'a Ident, ()> { + pub fn expect_ident_or_underscore(&mut self) -> Result<Ident, ()> { match self.expect_leaf()? { Leaf::Ident(it) => Ok(it), _ => Err(()), } } - pub fn expect_literal(&mut self) -> Result<&'a Leaf, ()> { + pub fn expect_literal(&mut self) -> Result<Leaf, ()> { let it = self.expect_leaf()?; - match it { + match &it { Leaf::Literal(_) => Ok(it), Leaf::Ident(ident) if ident.sym == sym::true_ || ident.sym == sym::false_ => Ok(it), _ => Err(()), } } - pub fn expect_single_punct(&mut self) -> Result<&'a Punct, ()> { + pub fn expect_single_punct(&mut self) -> Result<Punct, ()> { match self.expect_leaf()? { Leaf::Punct(it) => Ok(it), _ => Err(()), @@ -113,7 +113,7 @@ impl<'a> TtIter<'a> { /// This method currently may return a single quotation, which is part of lifetime ident and /// conceptually not a punct in the context of mbe. Callers should handle this. pub fn expect_glued_punct(&mut self) -> Result<ArrayVec<Punct, MAX_GLUED_PUNCT_LEN>, ()> { - let TtElement::Leaf(&Leaf::Punct(first)) = self.next().ok_or(())? else { + let TtElement::Leaf(Leaf::Punct(first)) = self.next().ok_or(())? else { return Err(()); }; @@ -168,11 +168,11 @@ impl<'a> TtIter<'a> { pub fn peek(&self) -> Option<TtElement<'a>> { match self.inner.as_slice().first()? { - TokenTree::Leaf(leaf) => Some(TtElement::Leaf(leaf)), + TokenTree::Leaf(leaf) => Some(TtElement::Leaf(leaf.clone())), TokenTree::Subtree(subtree) => { let nested_iter = TtIter { inner: self.inner.as_slice()[1..][..subtree.usize_len()].iter() }; - Some(TtElement::Subtree(subtree, nested_iter)) + Some(TtElement::Subtree(*subtree, nested_iter)) } } } @@ -214,8 +214,8 @@ impl<'a> TtIter<'a> { #[derive(Clone)] pub enum TtElement<'a> { - Leaf(&'a Leaf), - Subtree(&'a Subtree, TtIter<'a>), + Leaf(Leaf), + Subtree(Subtree, TtIter<'a>), } impl fmt::Debug for TtElement<'_> { @@ -243,12 +243,12 @@ impl<'a> Iterator for TtIter<'a> { type Item = TtElement<'a>; fn next(&mut self) -> Option<Self::Item> { match self.inner.next()? { - TokenTree::Leaf(leaf) => Some(TtElement::Leaf(leaf)), + TokenTree::Leaf(leaf) => Some(TtElement::Leaf(leaf.clone())), TokenTree::Subtree(subtree) => { let nested_iter = TtIter { inner: self.inner.as_slice()[..subtree.usize_len()].iter() }; self.inner = self.inner.as_slice()[subtree.usize_len()..].iter(); - Some(TtElement::Subtree(subtree, nested_iter)) + Some(TtElement::Subtree(*subtree, nested_iter)) } } } diff --git a/crates/tt/src/lib.rs b/crates/tt/src/lib.rs index 636f567f1a..97e0b34ad7 100644 --- a/crates/tt/src/lib.rs +++ b/crates/tt/src/lib.rs @@ -111,7 +111,7 @@ impl Leaf { } impl_from!(Literal, Punct, Ident for Leaf); -#[derive(Debug, Clone, PartialEq, Eq, Hash)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] pub struct Subtree { pub delimiter: Delimiter, /// Number of following token trees that belong to this subtree, excluding this subtree. @@ -125,7 +125,7 @@ impl Subtree { } #[derive(Clone, PartialEq, Eq, Hash)] -pub struct TopSubtree(pub Box<[TokenTree]>); +pub struct TopSubtree(Box<[TokenTree]>); impl TopSubtree { pub fn empty(span: DelimSpan) -> Self { @@ -147,6 +147,10 @@ impl TopSubtree { builder.build() } + pub fn from_serialized(tt: Vec<TokenTree>) -> Self { + Self(tt.into_boxed_slice()) + } + pub fn from_subtree(subtree: SubtreeView<'_>) -> Self { Self(subtree.0.into()) } @@ -159,20 +163,55 @@ impl TopSubtree { self.view().iter() } - pub fn top_subtree(&self) -> &Subtree { + pub fn top_subtree(&self) -> Subtree { self.view().top_subtree() } - pub fn top_subtree_delimiter_mut(&mut self) -> &mut Delimiter { + pub fn set_top_subtree_delimiter_kind(&mut self, kind: DelimiterKind) { + self.top_subtree_mut().delimiter.kind = kind; + } + + pub fn set_top_subtree_delimiter_span(&mut self, span: DelimSpan) { + let top_subtree = self.top_subtree_mut(); + top_subtree.delimiter.open = span.open; + top_subtree.delimiter.close = span.close; + } + + fn top_subtree_mut(&mut self) -> &mut Subtree { let TokenTree::Subtree(subtree) = &mut self.0[0] else { unreachable!("the first token tree is always the top subtree"); }; - &mut subtree.delimiter + subtree + } + + pub fn set_token(&mut self, idx: usize, leaf: Leaf) { + assert!(matches!(self.0[idx], TokenTree::Leaf(_)), "cannot replace a subtree by a leaf"); + self.0[idx] = leaf.into(); } pub fn token_trees(&self) -> TokenTreesView<'_> { self.view().token_trees() } + + pub fn as_token_trees(&self) -> TokenTreesView<'_> { + self.view().as_token_trees() + } + + pub fn change_every_ast_id(&mut self, mut callback: impl FnMut(&mut span::ErasedFileAstId)) { + for tt in &mut self.0 { + match tt { + TokenTree::Leaf(Leaf::Ident(Ident { span, .. })) + | TokenTree::Leaf(Leaf::Literal(Literal { span, .. })) + | TokenTree::Leaf(Leaf::Punct(Punct { span, .. })) => { + callback(&mut span.anchor.ast_id); + } + TokenTree::Subtree(subtree) => { + callback(&mut subtree.delimiter.open.anchor.ast_id); + callback(&mut subtree.delimiter.close.anchor.ast_id); + } + } + } + } } #[derive(Debug, Clone, PartialEq, Eq, Hash)] @@ -241,11 +280,6 @@ impl TopSubtreeBuilder { self.token_trees.extend(leaves.into_iter().map(TokenTree::Leaf)); } - /// This does not check the token trees are valid, beware! - pub fn extend_tt_dangerous(&mut self, tt: impl IntoIterator<Item = TokenTree>) { - self.token_trees.extend(tt); - } - pub fn extend_with_tt(&mut self, tt: TokenTreesView<'_>) { self.token_trees.extend(tt.0.iter().cloned()); } @@ -267,12 +301,12 @@ impl TopSubtreeBuilder { } } - pub fn expected_delimiters(&self) -> impl Iterator<Item = &Delimiter> { + pub fn expected_delimiters(&self) -> impl Iterator<Item = DelimiterKind> { self.unclosed_subtree_indices.iter().rev().map(|&subtree_idx| { let TokenTree::Subtree(subtree) = &self.token_trees[subtree_idx] else { unreachable!("unclosed token tree is always a subtree") }; - &subtree.delimiter + subtree.delimiter.kind }) } @@ -330,7 +364,7 @@ pub struct SubtreeBuilderRestorePoint { pub struct TokenTreesView<'a>(&'a [TokenTree]); impl<'a> TokenTreesView<'a> { - pub fn new(tts: &'a [TokenTree]) -> Self { + fn new(tts: &'a [TokenTree]) -> Self { if cfg!(debug_assertions) { tts.iter().enumerate().for_each(|(idx, tt)| { if let TokenTree::Subtree(tt) = &tt { @@ -345,12 +379,16 @@ impl<'a> TokenTreesView<'a> { Self(tts) } + pub fn empty() -> Self { + Self(&[]) + } + pub fn iter(&self) -> TtIter<'a> { TtIter::new(self.0) } pub fn cursor(&self) -> Cursor<'a> { - Cursor::new(self.0) + Cursor::new(*self) } pub fn len(&self) -> usize { @@ -374,13 +412,6 @@ impl<'a> TokenTreesView<'a> { self.try_into_subtree().map(|subtree| subtree.strip_invisible()).unwrap_or(self) } - /// This returns a **flat** structure of tokens (subtrees will be represented by a single node - /// preceding their children), so it isn't suited for most use cases, only for matching leaves - /// at the beginning/end with no subtrees before them. If you need a structured pass, use [`TtIter`]. - pub fn flat_tokens(&self) -> &'a [TokenTree] { - self.0 - } - pub fn split( self, mut split_fn: impl FnMut(TtElement<'a>) -> bool, @@ -406,6 +437,21 @@ impl<'a> TokenTreesView<'a> { Some(result) }) } + + pub fn first_span(&self) -> Option<Span> { + Some(self.0.first()?.first_span()) + } + + pub fn last_span(&self) -> Option<Span> { + Some(match self.0.last()? { + TokenTree::Leaf(it) => *it.span(), + TokenTree::Subtree(it) => it.delimiter.close, + }) + } + + pub fn iter_flat_tokens(&self) -> impl ExactSizeIterator<Item = TokenTree> + use<'a> { + self.0.iter().cloned() + } } impl fmt::Debug for TokenTreesView<'_> { @@ -453,11 +499,11 @@ impl fmt::Display for TokenTreesView<'_> { match child { TtElement::Leaf(Leaf::Punct(p)) => { needs_space = p.spacing == Spacing::Alone; - fmt::Display::fmt(p, f)?; + fmt::Display::fmt(&p, f)?; } - TtElement::Leaf(leaf) => fmt::Display::fmt(leaf, f)?, + TtElement::Leaf(leaf) => fmt::Display::fmt(&leaf, f)?, TtElement::Subtree(subtree, subtree_iter) => { - subtree_display(subtree, f, subtree_iter)? + subtree_display(&subtree, f, subtree_iter)? } } } @@ -493,11 +539,11 @@ impl<'a> SubtreeView<'a> { TtIter::new(&self.0[1..]) } - pub fn top_subtree(&self) -> &'a Subtree { + pub fn top_subtree(&self) -> Subtree { let TokenTree::Subtree(subtree) = &self.0[0] else { unreachable!("the first token tree is always the top subtree"); }; - subtree + *subtree } pub fn strip_invisible(&self) -> TokenTreesView<'a> { @@ -791,7 +837,7 @@ fn print_debug_token(f: &mut fmt::Formatter<'_>, level: usize, tt: TtElement<'_> } }, TtElement::Subtree(subtree, subtree_iter) => { - print_debug_subtree(f, subtree, level, subtree_iter)?; + print_debug_subtree(f, &subtree, level, subtree_iter)?; } } @@ -956,7 +1002,7 @@ impl TopSubtree { } } -pub fn pretty(mut tkns: &[TokenTree]) -> String { +pub fn pretty(tkns: TokenTreesView<'_>) -> String { fn tokentree_to_text(tkn: &TokenTree, tkns: &mut &[TokenTree]) -> String { match tkn { TokenTree::Leaf(Leaf::Ident(ident)) => { @@ -966,7 +1012,7 @@ pub fn pretty(mut tkns: &[TokenTree]) -> String { TokenTree::Leaf(Leaf::Punct(punct)) => format!("{}", punct.char), TokenTree::Subtree(subtree) => { let (subtree_content, rest) = tkns.split_at(subtree.usize_len()); - let content = pretty(subtree_content); + let content = pretty(TokenTreesView(subtree_content)); *tkns = rest; let (open, close) = match subtree.delimiter.kind { DelimiterKind::Brace => ("{", "}"), @@ -979,6 +1025,7 @@ pub fn pretty(mut tkns: &[TokenTree]) -> String { } } + let mut tkns = tkns.0; let mut last = String::new(); let mut last_to_joint = true; @@ -994,3 +1041,83 @@ pub fn pretty(mut tkns: &[TokenTree]) -> String { } last } + +#[derive(Debug)] +pub enum TransformTtAction<'a> { + Keep, + ReplaceWith(TokenTreesView<'a>), +} + +impl TransformTtAction<'_> { + #[inline] + pub fn remove() -> Self { + Self::ReplaceWith(TokenTreesView::empty()) + } +} + +/// This function takes a token tree, and calls `callback` with each token tree in it. +/// Then it does what the callback says: keeps the tt or replaces it with a (possibly empty) +/// tts view. +pub fn transform_tt<'b>( + tt: &mut TopSubtree, + mut callback: impl FnMut(TokenTree) -> TransformTtAction<'b>, +) { + let mut tt_vec = std::mem::take(&mut tt.0).into_vec(); + + // We need to keep a stack of the currently open subtrees, because we need to update + // them if we change the number of items in them. + let mut subtrees_stack = Vec::new(); + let mut i = 0; + while i < tt_vec.len() { + 'pop_finished_subtrees: while let Some(&subtree_idx) = subtrees_stack.last() { + let TokenTree::Subtree(subtree) = &tt_vec[subtree_idx] else { + unreachable!("non-subtree on subtrees stack"); + }; + if i >= subtree_idx + 1 + subtree.usize_len() { + subtrees_stack.pop(); + } else { + break 'pop_finished_subtrees; + } + } + + let current = match &tt_vec[i] { + TokenTree::Leaf(leaf) => TokenTree::Leaf(match leaf { + Leaf::Literal(leaf) => Leaf::Literal(leaf.clone()), + Leaf::Punct(leaf) => Leaf::Punct(*leaf), + Leaf::Ident(leaf) => Leaf::Ident(leaf.clone()), + }), + TokenTree::Subtree(subtree) => TokenTree::Subtree(*subtree), + }; + let action = callback(current); + match action { + TransformTtAction::Keep => { + // This cannot be shared with the replaced case, because then we may push the same subtree + // twice, and will update it twice which will lead to errors. + if let TokenTree::Subtree(_) = &tt_vec[i] { + subtrees_stack.push(i); + } + + i += 1; + } + TransformTtAction::ReplaceWith(replacement) => { + let old_len = 1 + match &tt_vec[i] { + TokenTree::Leaf(_) => 0, + TokenTree::Subtree(subtree) => subtree.usize_len(), + }; + let len_diff = replacement.len() as i64 - old_len as i64; + tt_vec.splice(i..i + old_len, replacement.0.iter().cloned()); + // Skip the newly inserted replacement, we don't want to visit it. + i += replacement.len(); + + for &subtree_idx in &subtrees_stack { + let TokenTree::Subtree(subtree) = &mut tt_vec[subtree_idx] else { + unreachable!("non-subtree on subtrees stack"); + }; + subtree.len = (i64::from(subtree.len) + len_diff).try_into().unwrap(); + } + } + } + } + + tt.0 = tt_vec.into_boxed_slice(); +} |