bendns' repos / rust-analyzer
Unnamed repository; edit this file 'description' to name the repository.
Diffstat (limited to 'crates/tt/src/lib.rs')
-rw-r--r--crates/tt/src/lib.rs927
1 files changed, 395 insertions, 532 deletions
diff --git a/crates/tt/src/lib.rs b/crates/tt/src/lib.rs
index f9a547f611..72b0d762ef 100644
--- a/crates/tt/src/lib.rs
+++ b/crates/tt/src/lib.rs
@@ -5,6 +5,9 @@
#![cfg_attr(feature = "in-rust-tree", feature(rustc_private))]
+#[cfg(feature = "in-rust-tree")]
+extern crate rustc_driver as _;
+
#[cfg(not(feature = "in-rust-tree"))]
extern crate ra_ap_rustc_lexer as rustc_lexer;
#[cfg(feature = "in-rust-tree")]
@@ -12,16 +15,23 @@ extern crate rustc_lexer;
pub mod buffer;
pub mod iter;
+mod storage;
-use std::fmt;
+use std::{fmt, slice::SliceIndex};
+use arrayvec::ArrayString;
use buffer::Cursor;
use intern::Symbol;
-use iter::{TtElement, TtIter};
use stdx::{impl_from, itertools::Itertools as _};
+pub use span::Span;
pub use text_size::{TextRange, TextSize};
+use crate::storage::{CompressedSpanPart, SpanStorage};
+
+pub use self::iter::{TtElement, TtIter};
+pub use self::storage::{TopSubtree, TopSubtreeBuilder};
+
pub const MAX_GLUED_PUNCT_LEN: usize = 3;
#[derive(Clone, PartialEq, Debug)]
@@ -74,13 +84,13 @@ pub enum LitKind {
}
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
-pub enum TokenTree<S = u32> {
- Leaf(Leaf<S>),
- Subtree(Subtree<S>),
+pub enum TokenTree {
+ Leaf(Leaf),
+ Subtree(Subtree),
}
-impl_from!(Leaf<S>, Subtree<S> for TokenTree);
-impl<S: Copy> TokenTree<S> {
- pub fn first_span(&self) -> S {
+impl_from!(Leaf, Subtree for TokenTree);
+impl TokenTree {
+ pub fn first_span(&self) -> Span {
match self {
TokenTree::Leaf(l) => *l.span(),
TokenTree::Subtree(s) => s.delimiter.open,
@@ -89,14 +99,14 @@ impl<S: Copy> TokenTree<S> {
}
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
-pub enum Leaf<S> {
- Literal(Literal<S>),
- Punct(Punct<S>),
- Ident(Ident<S>),
+pub enum Leaf {
+ Literal(Literal),
+ Punct(Punct),
+ Ident(Ident),
}
-impl<S> Leaf<S> {
- pub fn span(&self) -> &S {
+impl Leaf {
+ pub fn span(&self) -> &Span {
match self {
Leaf::Literal(it) => &it.span,
Leaf::Punct(it) => &it.span,
@@ -104,282 +114,120 @@ impl<S> Leaf<S> {
}
}
}
-impl_from!(Literal<S>, Punct<S>, Ident<S> for Leaf);
+impl_from!(Literal, Punct, Ident for Leaf);
-#[derive(Debug, Clone, PartialEq, Eq, Hash)]
-pub struct Subtree<S> {
- pub delimiter: Delimiter<S>,
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub struct Subtree {
+ pub delimiter: Delimiter,
/// Number of following token trees that belong to this subtree, excluding this subtree.
pub len: u32,
}
-impl<S> Subtree<S> {
+impl Subtree {
pub fn usize_len(&self) -> usize {
self.len as usize
}
}
-#[derive(Clone, PartialEq, Eq, Hash)]
-pub struct TopSubtree<S>(pub Box<[TokenTree<S>]>);
-
-impl<S: Copy> TopSubtree<S> {
- pub fn empty(span: DelimSpan<S>) -> Self {
- Self(Box::new([TokenTree::Subtree(Subtree {
- delimiter: Delimiter::invisible_delim_spanned(span),
- len: 0,
- })]))
- }
-
- pub fn invisible_from_leaves<const N: usize>(delim_span: S, leaves: [Leaf<S>; N]) -> Self {
- let mut builder = TopSubtreeBuilder::new(Delimiter::invisible_spanned(delim_span));
- builder.extend(leaves);
- builder.build()
- }
-
- pub fn from_token_trees(delimiter: Delimiter<S>, token_trees: TokenTreesView<'_, S>) -> Self {
- let mut builder = TopSubtreeBuilder::new(delimiter);
- builder.extend_with_tt(token_trees);
- builder.build()
- }
-
- pub fn from_subtree(subtree: SubtreeView<'_, S>) -> Self {
- Self(subtree.0.into())
- }
-
- pub fn view(&self) -> SubtreeView<'_, S> {
- SubtreeView::new(&self.0)
- }
-
- pub fn iter(&self) -> TtIter<'_, S> {
- self.view().iter()
- }
-
- pub fn top_subtree(&self) -> &Subtree<S> {
- self.view().top_subtree()
- }
-
- pub fn top_subtree_delimiter_mut(&mut self) -> &mut Delimiter<S> {
- let TokenTree::Subtree(subtree) = &mut self.0[0] else {
- unreachable!("the first token tree is always the top subtree");
- };
- &mut subtree.delimiter
- }
-
- pub fn token_trees(&self) -> TokenTreesView<'_, S> {
- self.view().token_trees()
- }
-}
-
-#[derive(Debug, Clone, PartialEq, Eq, Hash)]
-pub struct TopSubtreeBuilder<S> {
- unclosed_subtree_indices: Vec<usize>,
- token_trees: Vec<TokenTree<S>>,
- last_closed_subtree: Option<usize>,
-}
-
-impl<S: Copy> TopSubtreeBuilder<S> {
- pub fn new(top_delimiter: Delimiter<S>) -> Self {
- let mut result = Self {
- unclosed_subtree_indices: Vec::new(),
- token_trees: Vec::new(),
- last_closed_subtree: None,
- };
- let top_subtree = TokenTree::Subtree(Subtree { delimiter: top_delimiter, len: 0 });
- result.token_trees.push(top_subtree);
- result
- }
-
- pub fn open(&mut self, delimiter_kind: DelimiterKind, open_span: S) {
- self.unclosed_subtree_indices.push(self.token_trees.len());
- self.token_trees.push(TokenTree::Subtree(Subtree {
- delimiter: Delimiter {
- open: open_span,
- close: open_span, // Will be overwritten on close.
- kind: delimiter_kind,
- },
- len: 0,
- }));
- }
-
- pub fn close(&mut self, close_span: S) {
- let last_unclosed_index = self
- .unclosed_subtree_indices
- .pop()
- .expect("attempt to close a `tt::Subtree` when none is open");
- let subtree_len = (self.token_trees.len() - last_unclosed_index - 1) as u32;
- let TokenTree::Subtree(subtree) = &mut self.token_trees[last_unclosed_index] else {
- unreachable!("unclosed token tree is always a subtree");
- };
- subtree.len = subtree_len;
- subtree.delimiter.close = close_span;
- self.last_closed_subtree = Some(last_unclosed_index);
- }
-
- /// You cannot call this consecutively, it will only work once after close.
- pub fn remove_last_subtree_if_invisible(&mut self) {
- let Some(last_subtree_idx) = self.last_closed_subtree else { return };
- if let TokenTree::Subtree(Subtree {
- delimiter: Delimiter { kind: DelimiterKind::Invisible, .. },
- ..
- }) = self.token_trees[last_subtree_idx]
- {
- self.token_trees.remove(last_subtree_idx);
- self.last_closed_subtree = None;
- }
- }
-
- pub fn push(&mut self, leaf: Leaf<S>) {
- self.token_trees.push(TokenTree::Leaf(leaf));
- }
-
- pub fn extend(&mut self, leaves: impl IntoIterator<Item = Leaf<S>>) {
- self.token_trees.extend(leaves.into_iter().map(TokenTree::Leaf));
- }
-
- /// This does not check the token trees are valid, beware!
- pub fn extend_tt_dangerous(&mut self, tt: impl IntoIterator<Item = TokenTree<S>>) {
- self.token_trees.extend(tt);
- }
-
- pub fn extend_with_tt(&mut self, tt: TokenTreesView<'_, S>) {
- self.token_trees.extend(tt.0.iter().cloned());
- }
-
- /// Like [`Self::extend_with_tt()`], but makes sure the new tokens will never be
- /// joint with whatever comes after them.
- pub fn extend_with_tt_alone(&mut self, tt: TokenTreesView<'_, S>) {
- if let Some((last, before_last)) = tt.0.split_last() {
- self.token_trees.reserve(tt.0.len());
- self.token_trees.extend(before_last.iter().cloned());
- let last = if let TokenTree::Leaf(Leaf::Punct(last)) = last {
- let mut last = *last;
- last.spacing = Spacing::Alone;
- TokenTree::Leaf(Leaf::Punct(last))
- } else {
- last.clone()
- };
- self.token_trees.push(last);
+#[rust_analyzer::macro_style(braces)]
+macro_rules! dispatch_ref {
+ (
+ match $scrutinee:expr => $tt:ident => $body:expr
+ ) => {
+ match $scrutinee {
+ $crate::TokenTreesReprRef::SpanStorage32($tt) => $body,
+ $crate::TokenTreesReprRef::SpanStorage64($tt) => $body,
+ $crate::TokenTreesReprRef::SpanStorage96($tt) => $body,
}
- }
-
- pub fn expected_delimiters(&self) -> impl Iterator<Item = &Delimiter<S>> {
- self.unclosed_subtree_indices.iter().rev().map(|&subtree_idx| {
- let TokenTree::Subtree(subtree) = &self.token_trees[subtree_idx] else {
- unreachable!("unclosed token tree is always a subtree")
- };
- &subtree.delimiter
- })
- }
+ };
+}
+use dispatch_ref;
- /// Builds, and remove the top subtree if it has only one subtree child.
- pub fn build_skip_top_subtree(mut self) -> TopSubtree<S> {
- let top_tts = TokenTreesView::new(&self.token_trees[1..]);
- match top_tts.try_into_subtree() {
- Some(_) => {
- assert!(
- self.unclosed_subtree_indices.is_empty(),
- "attempt to build an unbalanced `TopSubtreeBuilder`"
- );
- TopSubtree(self.token_trees.drain(1..).collect())
+#[derive(Clone, Copy)]
+enum TokenTreesReprRef<'a> {
+ SpanStorage32(&'a [crate::storage::TokenTree<crate::storage::SpanStorage32>]),
+ SpanStorage64(&'a [crate::storage::TokenTree<crate::storage::SpanStorage64>]),
+ SpanStorage96(&'a [crate::storage::TokenTree<crate::storage::SpanStorage96>]),
+}
+
+impl<'a> TokenTreesReprRef<'a> {
+ #[inline]
+ fn get<I>(&self, index: I) -> Option<Self>
+ where
+ I: SliceIndex<
+ [crate::storage::TokenTree<crate::storage::SpanStorage32>],
+ Output = [crate::storage::TokenTree<crate::storage::SpanStorage32>],
+ >,
+ I: SliceIndex<
+ [crate::storage::TokenTree<crate::storage::SpanStorage64>],
+ Output = [crate::storage::TokenTree<crate::storage::SpanStorage64>],
+ >,
+ I: SliceIndex<
+ [crate::storage::TokenTree<crate::storage::SpanStorage96>],
+ Output = [crate::storage::TokenTree<crate::storage::SpanStorage96>],
+ >,
+ {
+ Some(match self {
+ TokenTreesReprRef::SpanStorage32(tt) => {
+ TokenTreesReprRef::SpanStorage32(tt.get(index)?)
}
- None => self.build(),
- }
- }
-
- pub fn build(mut self) -> TopSubtree<S> {
- assert!(
- self.unclosed_subtree_indices.is_empty(),
- "attempt to build an unbalanced `TopSubtreeBuilder`"
- );
- let total_len = self.token_trees.len() as u32;
- let TokenTree::Subtree(top_subtree) = &mut self.token_trees[0] else {
- unreachable!("first token tree is always a subtree");
- };
- top_subtree.len = total_len - 1;
- TopSubtree(self.token_trees.into_boxed_slice())
- }
-
- pub fn restore_point(&self) -> SubtreeBuilderRestorePoint {
- SubtreeBuilderRestorePoint {
- unclosed_subtree_indices_len: self.unclosed_subtree_indices.len(),
- token_trees_len: self.token_trees.len(),
- last_closed_subtree: self.last_closed_subtree,
- }
- }
-
- pub fn restore(&mut self, restore_point: SubtreeBuilderRestorePoint) {
- self.unclosed_subtree_indices.truncate(restore_point.unclosed_subtree_indices_len);
- self.token_trees.truncate(restore_point.token_trees_len);
- self.last_closed_subtree = restore_point.last_closed_subtree;
+ TokenTreesReprRef::SpanStorage64(tt) => {
+ TokenTreesReprRef::SpanStorage64(tt.get(index)?)
+ }
+ TokenTreesReprRef::SpanStorage96(tt) => {
+ TokenTreesReprRef::SpanStorage96(tt.get(index)?)
+ }
+ })
}
}
#[derive(Clone, Copy)]
-pub struct SubtreeBuilderRestorePoint {
- unclosed_subtree_indices_len: usize,
- token_trees_len: usize,
- last_closed_subtree: Option<usize>,
+pub struct TokenTreesView<'a> {
+ repr: TokenTreesReprRef<'a>,
+ span_parts: &'a [CompressedSpanPart],
}
-#[derive(Clone, Copy)]
-pub struct TokenTreesView<'a, S>(&'a [TokenTree<S>]);
-
-impl<'a, S: Copy> TokenTreesView<'a, S> {
- pub fn new(tts: &'a [TokenTree<S>]) -> Self {
- if cfg!(debug_assertions) {
- tts.iter().enumerate().for_each(|(idx, tt)| {
- if let TokenTree::Subtree(tt) = &tt {
- // `<` and not `<=` because `Subtree.len` does not include the subtree node itself.
- debug_assert!(
- idx + tt.usize_len() < tts.len(),
- "`TokenTreeView::new()` was given a cut-in-half list"
- );
- }
- });
- }
- Self(tts)
+impl<'a> TokenTreesView<'a> {
+ pub fn empty() -> Self {
+ Self { repr: TokenTreesReprRef::SpanStorage32(&[]), span_parts: &[] }
}
- pub fn iter(&self) -> TtIter<'a, S> {
- TtIter::new(self.0)
+ pub fn iter(&self) -> TtIter<'a> {
+ TtIter::new(*self)
}
- pub fn cursor(&self) -> Cursor<'a, S> {
- Cursor::new(self.0)
+ pub fn cursor(&self) -> Cursor<'a> {
+ Cursor::new(*self)
}
pub fn len(&self) -> usize {
- self.0.len()
+ dispatch_ref! {
+ match self.repr => tt => tt.len()
+ }
}
pub fn is_empty(&self) -> bool {
- self.0.is_empty()
+ self.len() == 0
}
- pub fn try_into_subtree(self) -> Option<SubtreeView<'a, S>> {
- if let Some(TokenTree::Subtree(subtree)) = self.0.first()
- && subtree.usize_len() == (self.0.len() - 1)
- {
- return Some(SubtreeView::new(self.0));
- }
- None
+ pub fn try_into_subtree(self) -> Option<SubtreeView<'a>> {
+ let is_subtree = dispatch_ref! {
+ match self.repr => tt => matches!(
+ tt.first(),
+ Some(crate::storage::TokenTree::Subtree { len, .. }) if (*len as usize) == (tt.len() - 1)
+ )
+ };
+ if is_subtree { Some(SubtreeView(self)) } else { None }
}
- pub fn strip_invisible(self) -> TokenTreesView<'a, S> {
+ pub fn strip_invisible(self) -> TokenTreesView<'a> {
self.try_into_subtree().map(|subtree| subtree.strip_invisible()).unwrap_or(self)
}
- /// This returns a **flat** structure of tokens (subtrees will be represented by a single node
- /// preceding their children), so it isn't suited for most use cases, only for matching leaves
- /// at the beginning/end with no subtrees before them. If you need a structured pass, use [`TtIter`].
- pub fn flat_tokens(&self) -> &'a [TokenTree<S>] {
- self.0
- }
-
pub fn split(
self,
- mut split_fn: impl FnMut(TtElement<'a, S>) -> bool,
- ) -> impl Iterator<Item = TokenTreesView<'a, S>> {
+ mut split_fn: impl FnMut(TtElement<'a>) -> bool,
+ ) -> impl Iterator<Item = TokenTreesView<'a>> {
let mut subtree_iter = self.iter();
let mut need_to_yield_even_if_empty = true;
@@ -401,9 +249,29 @@ impl<'a, S: Copy> TokenTreesView<'a, S> {
Some(result)
})
}
+
+ pub fn first_span(&self) -> Option<Span> {
+ Some(dispatch_ref! {
+ match self.repr => tt => tt.first()?.first_span().span(self.span_parts)
+ })
+ }
+
+ pub fn last_span(&self) -> Option<Span> {
+ Some(dispatch_ref! {
+ match self.repr => tt => tt.last()?.last_span().span(self.span_parts)
+ })
+ }
+
+ pub fn iter_flat_tokens(self) -> impl ExactSizeIterator<Item = TokenTree> + use<'a> {
+ (0..self.len()).map(move |idx| {
+ dispatch_ref! {
+ match self.repr => tt => tt[idx].to_api(self.span_parts)
+ }
+ })
+ }
}
-impl<S: fmt::Debug + Copy> fmt::Debug for TokenTreesView<'_, S> {
+impl fmt::Debug for TokenTreesView<'_> {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
let mut iter = self.iter();
while let Some(tt) = iter.next() {
@@ -416,14 +284,14 @@ impl<S: fmt::Debug + Copy> fmt::Debug for TokenTreesView<'_, S> {
}
}
-impl<S: Copy> fmt::Display for TokenTreesView<'_, S> {
+impl fmt::Display for TokenTreesView<'_> {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
return token_trees_display(f, self.iter());
- fn subtree_display<S>(
- subtree: &Subtree<S>,
+ fn subtree_display(
+ subtree: &Subtree,
f: &mut fmt::Formatter<'_>,
- iter: TtIter<'_, S>,
+ iter: TtIter<'_>,
) -> fmt::Result {
let (l, r) = match subtree.delimiter.kind {
DelimiterKind::Parenthesis => ("(", ")"),
@@ -437,7 +305,7 @@ impl<S: Copy> fmt::Display for TokenTreesView<'_, S> {
Ok(())
}
- fn token_trees_display<S>(f: &mut fmt::Formatter<'_>, iter: TtIter<'_, S>) -> fmt::Result {
+ fn token_trees_display(f: &mut fmt::Formatter<'_>, iter: TtIter<'_>) -> fmt::Result {
let mut needs_space = false;
for child in iter {
if needs_space {
@@ -448,11 +316,11 @@ impl<S: Copy> fmt::Display for TokenTreesView<'_, S> {
match child {
TtElement::Leaf(Leaf::Punct(p)) => {
needs_space = p.spacing == Spacing::Alone;
- fmt::Display::fmt(p, f)?;
+ fmt::Display::fmt(&p, f)?;
}
- TtElement::Leaf(leaf) => fmt::Display::fmt(leaf, f)?,
+ TtElement::Leaf(leaf) => fmt::Display::fmt(&leaf, f)?,
TtElement::Subtree(subtree, subtree_iter) => {
- subtree_display(subtree, f, subtree_iter)?
+ subtree_display(&subtree, f, subtree_iter)?
}
}
}
@@ -463,70 +331,80 @@ impl<S: Copy> fmt::Display for TokenTreesView<'_, S> {
#[derive(Clone, Copy)]
// Invariant: always starts with `Subtree` that covers the entire thing.
-pub struct SubtreeView<'a, S>(&'a [TokenTree<S>]);
-
-impl<'a, S: Copy> SubtreeView<'a, S> {
- pub fn new(tts: &'a [TokenTree<S>]) -> Self {
- if cfg!(debug_assertions) {
- let TokenTree::Subtree(subtree) = &tts[0] else {
- panic!("first token tree must be a subtree in `SubtreeView`");
- };
- assert_eq!(
- subtree.usize_len(),
- tts.len() - 1,
- "subtree must cover the entire `SubtreeView`"
- );
- }
- Self(tts)
- }
+pub struct SubtreeView<'a>(TokenTreesView<'a>);
- pub fn as_token_trees(self) -> TokenTreesView<'a, S> {
- TokenTreesView::new(self.0)
+impl<'a> SubtreeView<'a> {
+ pub fn as_token_trees(self) -> TokenTreesView<'a> {
+ self.0
}
- pub fn iter(&self) -> TtIter<'a, S> {
- TtIter::new(&self.0[1..])
+ pub fn iter(&self) -> TtIter<'a> {
+ self.token_trees().iter()
}
- pub fn top_subtree(&self) -> &'a Subtree<S> {
- let TokenTree::Subtree(subtree) = &self.0[0] else {
- unreachable!("the first token tree is always the top subtree");
- };
- subtree
+ pub fn top_subtree(&self) -> Subtree {
+ dispatch_ref! {
+ match self.0.repr => tt => {
+ let crate::storage::TokenTree::Subtree { len, delim_kind, open_span, close_span } =
+ &tt[0]
+ else {
+ unreachable!("the first token tree is always the top subtree");
+ };
+ Subtree {
+ delimiter: Delimiter {
+ open: open_span.span(self.0.span_parts),
+ close: close_span.span(self.0.span_parts),
+ kind: *delim_kind,
+ },
+ len: *len,
+ }
+ }
+ }
}
- pub fn strip_invisible(&self) -> TokenTreesView<'a, S> {
+ pub fn strip_invisible(&self) -> TokenTreesView<'a> {
if self.top_subtree().delimiter.kind == DelimiterKind::Invisible {
- TokenTreesView::new(&self.0[1..])
+ self.token_trees()
} else {
- TokenTreesView::new(self.0)
+ self.0
}
}
- pub fn token_trees(&self) -> TokenTreesView<'a, S> {
- TokenTreesView::new(&self.0[1..])
+ pub fn token_trees(&self) -> TokenTreesView<'a> {
+ let repr = match self.0.repr {
+ TokenTreesReprRef::SpanStorage32(token_trees) => {
+ TokenTreesReprRef::SpanStorage32(&token_trees[1..])
+ }
+ TokenTreesReprRef::SpanStorage64(token_trees) => {
+ TokenTreesReprRef::SpanStorage64(&token_trees[1..])
+ }
+ TokenTreesReprRef::SpanStorage96(token_trees) => {
+ TokenTreesReprRef::SpanStorage96(&token_trees[1..])
+ }
+ };
+ TokenTreesView { repr, ..self.0 }
}
}
-impl<S: fmt::Debug + Copy> fmt::Debug for SubtreeView<'_, S> {
+impl fmt::Debug for SubtreeView<'_> {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
- fmt::Debug::fmt(&TokenTreesView(self.0), f)
+ fmt::Debug::fmt(&self.0, f)
}
}
-impl<S: Copy> fmt::Display for SubtreeView<'_, S> {
+impl fmt::Display for SubtreeView<'_> {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
- fmt::Display::fmt(&TokenTreesView(self.0), f)
+ fmt::Display::fmt(&self.0, f)
}
}
#[derive(Debug, Copy, Clone, PartialEq)]
-pub struct DelimSpan<S> {
- pub open: S,
- pub close: S,
+pub struct DelimSpan {
+ pub open: Span,
+ pub close: Span,
}
-impl<Span: Copy> DelimSpan<Span> {
+impl DelimSpan {
pub fn from_single(sp: Span) -> Self {
DelimSpan { open: sp, close: sp }
}
@@ -536,22 +414,22 @@ impl<Span: Copy> DelimSpan<Span> {
}
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
-pub struct Delimiter<S> {
- pub open: S,
- pub close: S,
+pub struct Delimiter {
+ pub open: Span,
+ pub close: Span,
pub kind: DelimiterKind,
}
-impl<S: Copy> Delimiter<S> {
- pub const fn invisible_spanned(span: S) -> Self {
+impl Delimiter {
+ pub const fn invisible_spanned(span: Span) -> Self {
Delimiter { open: span, close: span, kind: DelimiterKind::Invisible }
}
- pub const fn invisible_delim_spanned(span: DelimSpan<S>) -> Self {
+ pub const fn invisible_delim_spanned(span: DelimSpan) -> Self {
Delimiter { open: span.open, close: span.close, kind: DelimiterKind::Invisible }
}
- pub fn delim_span(&self) -> DelimSpan<S> {
+ pub fn delim_span(&self) -> DelimSpan {
DelimSpan { open: self.open, close: self.close }
}
}
@@ -565,18 +443,57 @@ pub enum DelimiterKind {
}
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
-pub struct Literal<S> {
- // escaped
- pub symbol: Symbol,
- pub span: S,
+pub struct Literal {
+ /// Escaped, text then suffix concatenated.
+ pub text_and_suffix: Symbol,
+ pub span: Span,
pub kind: LitKind,
- pub suffix: Option<Symbol>,
+ pub suffix_len: u8,
}
-pub fn token_to_literal<S>(text: &str, span: S) -> Literal<S>
-where
- S: Copy,
-{
+impl Literal {
+ #[inline]
+ pub fn text_and_suffix(&self) -> (&str, &str) {
+ let text_and_suffix = self.text_and_suffix.as_str();
+ text_and_suffix.split_at(text_and_suffix.len() - usize::from(self.suffix_len))
+ }
+
+ #[inline]
+ pub fn text(&self) -> &str {
+ self.text_and_suffix().0
+ }
+
+ #[inline]
+ pub fn suffix(&self) -> &str {
+ self.text_and_suffix().1
+ }
+
+ pub fn new(text: &str, span: Span, kind: LitKind, suffix: &str) -> Self {
+ const MAX_INLINE_CAPACITY: usize = 30;
+ let text_and_suffix = if suffix.is_empty() {
+ Symbol::intern(text)
+ } else if (text.len() + suffix.len()) < MAX_INLINE_CAPACITY {
+ let mut text_and_suffix = ArrayString::<MAX_INLINE_CAPACITY>::new();
+ text_and_suffix.push_str(text);
+ text_and_suffix.push_str(suffix);
+ Symbol::intern(&text_and_suffix)
+ } else {
+ let mut text_and_suffix = String::with_capacity(text.len() + suffix.len());
+ text_and_suffix.push_str(text);
+ text_and_suffix.push_str(suffix);
+ Symbol::intern(&text_and_suffix)
+ };
+
+ Self { text_and_suffix, span, kind, suffix_len: suffix.len().try_into().unwrap() }
+ }
+
+ #[inline]
+ pub fn new_no_suffix(text: &str, span: Span, kind: LitKind) -> Self {
+ Self { text_and_suffix: Symbol::intern(text), span, kind, suffix_len: 0 }
+ }
+}
+
+pub fn token_to_literal(text: &str, span: Span) -> Literal {
use rustc_lexer::LiteralKind;
let token = rustc_lexer::tokenize(text, rustc_lexer::FrontmatterAllowed::No).next_tuple();
@@ -585,12 +502,7 @@ where
..
},)) = token
else {
- return Literal {
- span,
- symbol: Symbol::intern(text),
- kind: LitKind::Err(()),
- suffix: None,
- };
+ return Literal::new_no_suffix(text, span, LitKind::Err(()));
};
let (kind, start_offset, end_offset) = match kind {
@@ -621,27 +533,22 @@ where
let (lit, suffix) = text.split_at(suffix_start as usize);
let lit = &lit[start_offset..lit.len() - end_offset];
let suffix = match suffix {
- "" | "_" => None,
+ "" | "_" => "",
// ill-suffixed literals
_ if !matches!(kind, LitKind::Integer | LitKind::Float | LitKind::Err(_)) => {
- return Literal {
- span,
- symbol: Symbol::intern(text),
- kind: LitKind::Err(()),
- suffix: None,
- };
+ return Literal::new_no_suffix(text, span, LitKind::Err(()));
}
- suffix => Some(Symbol::intern(suffix)),
+ suffix => suffix,
};
- Literal { span, symbol: Symbol::intern(lit), kind, suffix }
+ Literal::new(lit, span, kind, suffix)
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
-pub struct Punct<S> {
+pub struct Punct {
pub char: char,
pub spacing: Spacing,
- pub span: S,
+ pub span: Span,
}
/// Indicates whether a token can join with the following token to form a
@@ -706,25 +613,25 @@ pub enum Spacing {
/// Identifier or keyword.
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
-pub struct Ident<S> {
+pub struct Ident {
pub sym: Symbol,
- pub span: S,
+ pub span: Span,
pub is_raw: IdentIsRaw,
}
-impl<S> Ident<S> {
- pub fn new(text: &str, span: S) -> Self {
+impl Ident {
+ pub fn new(text: &str, span: Span) -> Self {
// let raw_stripped = IdentIsRaw::split_from_symbol(text.as_ref());
let (is_raw, text) = IdentIsRaw::split_from_symbol(text);
Ident { sym: Symbol::intern(text), span, is_raw }
}
}
-fn print_debug_subtree<S: fmt::Debug>(
+fn print_debug_subtree(
f: &mut fmt::Formatter<'_>,
- subtree: &Subtree<S>,
+ subtree: &Subtree,
level: usize,
- iter: TtIter<'_, S>,
+ iter: TtIter<'_>,
) -> fmt::Result {
let align = " ".repeat(level);
@@ -748,25 +655,14 @@ fn print_debug_subtree<S: fmt::Debug>(
Ok(())
}
-fn print_debug_token<S: fmt::Debug>(
- f: &mut fmt::Formatter<'_>,
- level: usize,
- tt: TtElement<'_, S>,
-) -> fmt::Result {
+fn print_debug_token(f: &mut fmt::Formatter<'_>, level: usize, tt: TtElement<'_>) -> fmt::Result {
let align = " ".repeat(level);
match tt {
TtElement::Leaf(leaf) => match leaf {
Leaf::Literal(lit) => {
- write!(
- f,
- "{}LITERAL {:?} {}{} {:#?}",
- align,
- lit.kind,
- lit.symbol,
- lit.suffix.as_ref().map(|it| it.as_str()).unwrap_or(""),
- lit.span
- )?;
+ let (text, suffix) = lit.text_and_suffix();
+ write!(f, "{}LITERAL {:?} {}{} {:#?}", align, lit.kind, text, suffix, lit.span)?;
}
Leaf::Punct(punct) => {
write!(
@@ -790,26 +686,26 @@ fn print_debug_token<S: fmt::Debug>(
}
},
TtElement::Subtree(subtree, subtree_iter) => {
- print_debug_subtree(f, subtree, level, subtree_iter)?;
+ print_debug_subtree(f, &subtree, level, subtree_iter)?;
}
}
Ok(())
}
-impl<S: fmt::Debug + Copy> fmt::Debug for TopSubtree<S> {
+impl fmt::Debug for TopSubtree {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
fmt::Debug::fmt(&self.view(), f)
}
}
-impl<S: fmt::Display + Copy> fmt::Display for TopSubtree<S> {
+impl fmt::Display for TopSubtree {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
fmt::Display::fmt(&self.view(), f)
}
}
-impl<S> fmt::Display for Leaf<S> {
+impl fmt::Display for Leaf {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Leaf::Ident(it) => fmt::Display::fmt(it, f),
@@ -819,207 +715,88 @@ impl<S> fmt::Display for Leaf<S> {
}
}
-impl<S> fmt::Display for Ident<S> {
+impl fmt::Display for Ident {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
fmt::Display::fmt(&self.is_raw.as_str(), f)?;
fmt::Display::fmt(&self.sym, f)
}
}
-impl<S> Literal<S> {
- pub fn display_no_minus(&self) -> impl fmt::Display {
- struct NoMinus<'a, S>(&'a Literal<S>);
- impl<S> fmt::Display for NoMinus<'_, S> {
- fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
- let symbol =
- self.0.symbol.as_str().strip_prefix('-').unwrap_or(self.0.symbol.as_str());
- match self.0.kind {
- LitKind::Byte => write!(f, "b'{symbol}'"),
- LitKind::Char => write!(f, "'{symbol}'"),
- LitKind::Integer | LitKind::Float | LitKind::Err(_) => write!(f, "{symbol}"),
- LitKind::Str => write!(f, "\"{symbol}\""),
- LitKind::ByteStr => write!(f, "b\"{symbol}\""),
- LitKind::CStr => write!(f, "c\"{symbol}\""),
- LitKind::StrRaw(num_of_hashes) => {
- let num_of_hashes = num_of_hashes as usize;
- write!(
- f,
- r#"r{0:#<num_of_hashes$}"{text}"{0:#<num_of_hashes$}"#,
- "",
- text = symbol
- )
- }
- LitKind::ByteStrRaw(num_of_hashes) => {
- let num_of_hashes = num_of_hashes as usize;
- write!(
- f,
- r#"br{0:#<num_of_hashes$}"{text}"{0:#<num_of_hashes$}"#,
- "",
- text = symbol
- )
- }
- LitKind::CStrRaw(num_of_hashes) => {
- let num_of_hashes = num_of_hashes as usize;
- write!(
- f,
- r#"cr{0:#<num_of_hashes$}"{text}"{0:#<num_of_hashes$}"#,
- "",
- text = symbol
- )
- }
- }?;
- if let Some(suffix) = &self.0.suffix {
- write!(f, "{suffix}")?;
- }
- Ok(())
- }
- }
- NoMinus(self)
- }
-}
-
-impl<S> fmt::Display for Literal<S> {
+impl fmt::Display for Literal {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+ let (text, suffix) = self.text_and_suffix();
match self.kind {
- LitKind::Byte => write!(f, "b'{}'", self.symbol),
- LitKind::Char => write!(f, "'{}'", self.symbol),
- LitKind::Integer | LitKind::Float | LitKind::Err(_) => write!(f, "{}", self.symbol),
- LitKind::Str => write!(f, "\"{}\"", self.symbol),
- LitKind::ByteStr => write!(f, "b\"{}\"", self.symbol),
- LitKind::CStr => write!(f, "c\"{}\"", self.symbol),
+ LitKind::Byte => write!(f, "b'{}'", text),
+ LitKind::Char => write!(f, "'{}'", text),
+ LitKind::Integer | LitKind::Float | LitKind::Err(_) => write!(f, "{}", text),
+ LitKind::Str => write!(f, "\"{}\"", text),
+ LitKind::ByteStr => write!(f, "b\"{}\"", text),
+ LitKind::CStr => write!(f, "c\"{}\"", text),
LitKind::StrRaw(num_of_hashes) => {
let num_of_hashes = num_of_hashes as usize;
- write!(
- f,
- r#"r{0:#<num_of_hashes$}"{text}"{0:#<num_of_hashes$}"#,
- "",
- text = self.symbol
- )
+ write!(f, r#"r{0:#<num_of_hashes$}"{text}"{0:#<num_of_hashes$}"#, "", text = text)
}
LitKind::ByteStrRaw(num_of_hashes) => {
let num_of_hashes = num_of_hashes as usize;
- write!(
- f,
- r#"br{0:#<num_of_hashes$}"{text}"{0:#<num_of_hashes$}"#,
- "",
- text = self.symbol
- )
+ write!(f, r#"br{0:#<num_of_hashes$}"{text}"{0:#<num_of_hashes$}"#, "", text = text)
}
LitKind::CStrRaw(num_of_hashes) => {
let num_of_hashes = num_of_hashes as usize;
- write!(
- f,
- r#"cr{0:#<num_of_hashes$}"{text}"{0:#<num_of_hashes$}"#,
- "",
- text = self.symbol
- )
+ write!(f, r#"cr{0:#<num_of_hashes$}"{text}"{0:#<num_of_hashes$}"#, "", text = text)
}
}?;
- if let Some(suffix) = &self.suffix {
- write!(f, "{suffix}")?;
- }
+ write!(f, "{suffix}")?;
Ok(())
}
}
-impl<S> fmt::Display for Punct<S> {
+impl fmt::Display for Punct {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
fmt::Display::fmt(&self.char, f)
}
}
-impl<S> Subtree<S> {
+impl Subtree {
/// Count the number of tokens recursively
pub fn count(&self) -> usize {
self.usize_len()
}
}
-impl<S> TopSubtree<S> {
- /// A simple line string used for debugging
- pub fn subtree_as_debug_string(&self, subtree_idx: usize) -> String {
- fn debug_subtree<S>(
- output: &mut String,
- subtree: &Subtree<S>,
- iter: &mut std::slice::Iter<'_, TokenTree<S>>,
- ) {
- let delim = match subtree.delimiter.kind {
- DelimiterKind::Brace => ("{", "}"),
- DelimiterKind::Bracket => ("[", "]"),
- DelimiterKind::Parenthesis => ("(", ")"),
- DelimiterKind::Invisible => ("$", "$"),
- };
-
- output.push_str(delim.0);
- let mut last = None;
- let mut idx = 0;
- while idx < subtree.len {
- let child = iter.next().unwrap();
- debug_token_tree(output, child, last, iter);
- last = Some(child);
- idx += 1;
- }
-
- output.push_str(delim.1);
- }
-
- fn debug_token_tree<S>(
- output: &mut String,
- tt: &TokenTree<S>,
- last: Option<&TokenTree<S>>,
- iter: &mut std::slice::Iter<'_, TokenTree<S>>,
- ) {
- match tt {
- TokenTree::Leaf(it) => {
- let s = match it {
- Leaf::Literal(it) => it.symbol.to_string(),
- Leaf::Punct(it) => it.char.to_string(),
- Leaf::Ident(it) => format!("{}{}", it.is_raw.as_str(), it.sym),
- };
- match (it, last) {
- (Leaf::Ident(_), Some(&TokenTree::Leaf(Leaf::Ident(_)))) => {
- output.push(' ');
- output.push_str(&s);
- }
- (Leaf::Punct(_), Some(TokenTree::Leaf(Leaf::Punct(punct)))) => {
- if punct.spacing == Spacing::Alone {
- output.push(' ');
- output.push_str(&s);
- } else {
- output.push_str(&s);
- }
- }
- _ => output.push_str(&s),
- }
- }
- TokenTree::Subtree(it) => debug_subtree(output, it, iter),
- }
- }
+pub fn pretty(tkns: TokenTreesView<'_>) -> String {
+ return dispatch_ref! {
+ match tkns.repr => tt => pretty_impl(tt)
+ };
- let mut res = String::new();
- debug_token_tree(
- &mut res,
- &self.0[subtree_idx],
- None,
- &mut self.0[subtree_idx + 1..].iter(),
- );
- res
- }
-}
+ use crate::storage::TokenTree;
-pub fn pretty<S>(mut tkns: &[TokenTree<S>]) -> String {
- fn tokentree_to_text<S>(tkn: &TokenTree<S>, tkns: &mut &[TokenTree<S>]) -> String {
+ fn tokentree_to_text<S: SpanStorage>(tkn: &TokenTree<S>, tkns: &mut &[TokenTree<S>]) -> String {
match tkn {
- TokenTree::Leaf(Leaf::Ident(ident)) => {
- format!("{}{}", ident.is_raw.as_str(), ident.sym)
+ TokenTree::Ident { sym, is_raw, .. } => format!("{}{}", is_raw.as_str(), sym),
+ &TokenTree::Literal { ref text_and_suffix, kind, suffix_len, span: _ } => {
+ format!(
+ "{}",
+ Literal {
+ text_and_suffix: text_and_suffix.clone(),
+ span: Span {
+ range: TextRange::empty(TextSize::new(0)),
+ anchor: span::SpanAnchor {
+ file_id: span::EditionedFileId::from_raw(0),
+ ast_id: span::FIXUP_ERASED_FILE_AST_ID_MARKER
+ },
+ ctx: span::SyntaxContext::root(span::Edition::Edition2015)
+ },
+ kind,
+ suffix_len
+ }
+ )
}
- TokenTree::Leaf(Leaf::Literal(literal)) => format!("{literal}"),
- TokenTree::Leaf(Leaf::Punct(punct)) => format!("{}", punct.char),
- TokenTree::Subtree(subtree) => {
- let (subtree_content, rest) = tkns.split_at(subtree.usize_len());
- let content = pretty(subtree_content);
+ TokenTree::Punct { char, .. } => format!("{}", char),
+ TokenTree::Subtree { len, delim_kind, .. } => {
+ let (subtree_content, rest) = tkns.split_at(*len as usize);
+ let content = pretty_impl(subtree_content);
*tkns = rest;
- let (open, close) = match subtree.delimiter.kind {
+ let (open, close) = match *delim_kind {
DelimiterKind::Brace => ("{", "}"),
DelimiterKind::Bracket => ("[", "]"),
DelimiterKind::Parenthesis => ("(", ")"),
@@ -1030,18 +807,104 @@ pub fn pretty<S>(mut tkns: &[TokenTree<S>]) -> String {
}
}
- let mut last = String::new();
- let mut last_to_joint = true;
-
- while let Some((tkn, rest)) = tkns.split_first() {
- tkns = rest;
- last = [last, tokentree_to_text(tkn, &mut tkns)].join(if last_to_joint { "" } else { " " });
- last_to_joint = false;
- if let TokenTree::Leaf(Leaf::Punct(punct)) = tkn
- && punct.spacing == Spacing::Joint
- {
- last_to_joint = true;
+ fn pretty_impl<S: SpanStorage>(mut tkns: &[TokenTree<S>]) -> String {
+ let mut last = String::new();
+ let mut last_to_joint = true;
+
+ while let Some((tkn, rest)) = tkns.split_first() {
+ tkns = rest;
+ last = [last, tokentree_to_text(tkn, &mut tkns)].join(if last_to_joint {
+ ""
+ } else {
+ " "
+ });
+ last_to_joint = false;
+ if let TokenTree::Punct { spacing, .. } = tkn
+ && *spacing == Spacing::Joint
+ {
+ last_to_joint = true;
+ }
}
+ last
}
- last
+}
+
+#[derive(Debug)]
+pub enum TransformTtAction<'a> {
+ Keep,
+ ReplaceWith(TokenTreesView<'a>),
+}
+
+impl TransformTtAction<'_> {
+ #[inline]
+ pub fn remove() -> Self {
+ Self::ReplaceWith(TokenTreesView::empty())
+ }
+}
+
+/// This function takes a token tree, and calls `callback` with each token tree in it.
+/// Then it does what the callback says: keeps the tt or replaces it with a (possibly empty)
+/// tts view.
+pub fn transform_tt<'b>(
+ tt: &mut TopSubtree,
+ mut callback: impl FnMut(TokenTree) -> TransformTtAction<'b>,
+) {
+ let mut tt_vec = tt.as_token_trees().iter_flat_tokens().collect::<Vec<_>>();
+
+ // We need to keep a stack of the currently open subtrees, because we need to update
+ // them if we change the number of items in them.
+ let mut subtrees_stack = Vec::new();
+ let mut i = 0;
+ while i < tt_vec.len() {
+ 'pop_finished_subtrees: while let Some(&subtree_idx) = subtrees_stack.last() {
+ let TokenTree::Subtree(subtree) = &tt_vec[subtree_idx] else {
+ unreachable!("non-subtree on subtrees stack");
+ };
+ if i >= subtree_idx + 1 + subtree.usize_len() {
+ subtrees_stack.pop();
+ } else {
+ break 'pop_finished_subtrees;
+ }
+ }
+
+ let current = match &tt_vec[i] {
+ TokenTree::Leaf(leaf) => TokenTree::Leaf(match leaf {
+ Leaf::Literal(leaf) => Leaf::Literal(leaf.clone()),
+ Leaf::Punct(leaf) => Leaf::Punct(*leaf),
+ Leaf::Ident(leaf) => Leaf::Ident(leaf.clone()),
+ }),
+ TokenTree::Subtree(subtree) => TokenTree::Subtree(*subtree),
+ };
+ let action = callback(current);
+ match action {
+ TransformTtAction::Keep => {
+ // This cannot be shared with the replaced case, because then we may push the same subtree
+ // twice, and will update it twice which will lead to errors.
+ if let TokenTree::Subtree(_) = &tt_vec[i] {
+ subtrees_stack.push(i);
+ }
+
+ i += 1;
+ }
+ TransformTtAction::ReplaceWith(replacement) => {
+ let old_len = 1 + match &tt_vec[i] {
+ TokenTree::Leaf(_) => 0,
+ TokenTree::Subtree(subtree) => subtree.usize_len(),
+ };
+ let len_diff = replacement.len() as i64 - old_len as i64;
+ tt_vec.splice(i..i + old_len, replacement.iter_flat_tokens());
+ // Skip the newly inserted replacement, we don't want to visit it.
+ i += replacement.len();
+
+ for &subtree_idx in &subtrees_stack {
+ let TokenTree::Subtree(subtree) = &mut tt_vec[subtree_idx] else {
+ unreachable!("non-subtree on subtrees stack");
+ };
+ subtree.len = (i64::from(subtree.len) + len_diff).try_into().unwrap();
+ }
+ }
+ }
+ }
+
+ *tt = TopSubtree::from_serialized(tt_vec);
}