Unnamed repository; edit this file 'description' to name the repository.
Diffstat (limited to 'crates/syntax-bridge/src/lib.rs')
| -rw-r--r-- | crates/syntax-bridge/src/lib.rs | 272 |
1 files changed, 118 insertions, 154 deletions
diff --git a/crates/syntax-bridge/src/lib.rs b/crates/syntax-bridge/src/lib.rs index 815b4f2799..0dcf18a4ad 100644 --- a/crates/syntax-bridge/src/lib.rs +++ b/crates/syntax-bridge/src/lib.rs @@ -9,7 +9,7 @@ use std::{collections::VecDeque, fmt, hash::Hash}; use intern::Symbol; use rustc_hash::{FxHashMap, FxHashSet}; -use span::{Edition, SpanAnchor, SpanData, SpanMap}; +use span::{Edition, Span, SpanAnchor, SpanMap, SyntaxContext}; use stdx::{format_to, never}; use syntax::{ AstToken, Parse, PreorderWithTokens, SmolStr, SyntaxElement, @@ -29,21 +29,18 @@ pub use ::parser::TopEntryPoint; #[cfg(test)] mod tests; -pub trait SpanMapper<S> { - fn span_for(&self, range: TextRange) -> S; +pub trait SpanMapper { + fn span_for(&self, range: TextRange) -> Span; } -impl<S> SpanMapper<SpanData<S>> for SpanMap<S> -where - SpanData<S>: Copy, -{ - fn span_for(&self, range: TextRange) -> SpanData<S> { +impl SpanMapper for SpanMap { + fn span_for(&self, range: TextRange) -> Span { self.span_at(range.start()) } } -impl<S: Copy, SM: SpanMapper<S>> SpanMapper<S> for &SM { - fn span_for(&self, range: TextRange) -> S { +impl<SM: SpanMapper> SpanMapper for &SM { + fn span_for(&self, range: TextRange) -> Span { SM::span_for(self, range) } } @@ -69,7 +66,7 @@ pub mod dummy_test_span_utils { pub struct DummyTestSpanMap; - impl SpanMapper<Span> for DummyTestSpanMap { + impl SpanMapper for DummyTestSpanMap { fn span_for(&self, range: syntax::TextRange) -> Span { Span { range, @@ -97,15 +94,14 @@ pub enum DocCommentDesugarMode { /// Converts a syntax tree to a [`tt::Subtree`] using the provided span map to populate the /// subtree's spans. -pub fn syntax_node_to_token_tree<Ctx, SpanMap>( +pub fn syntax_node_to_token_tree<SpanMap>( node: &SyntaxNode, map: SpanMap, - span: SpanData<Ctx>, + span: Span, mode: DocCommentDesugarMode, -) -> tt::TopSubtree<SpanData<Ctx>> +) -> tt::TopSubtree where - SpanData<Ctx>: Copy + fmt::Debug, - SpanMap: SpanMapper<SpanData<Ctx>>, + SpanMap: SpanMapper, { let mut c = Converter::new(node, map, Default::default(), Default::default(), span, mode, |_, _| { @@ -117,22 +113,18 @@ where /// Converts a syntax tree to a [`tt::Subtree`] using the provided span map to populate the /// subtree's spans. Additionally using the append and remove parameters, the additional tokens can /// be injected or hidden from the output. -pub fn syntax_node_to_token_tree_modified<Ctx, SpanMap, OnEvent>( +pub fn syntax_node_to_token_tree_modified<SpanMap, OnEvent>( node: &SyntaxNode, map: SpanMap, - append: FxHashMap<SyntaxElement, Vec<tt::Leaf<SpanData<Ctx>>>>, + append: FxHashMap<SyntaxElement, Vec<tt::Leaf>>, remove: FxHashSet<SyntaxElement>, - call_site: SpanData<Ctx>, + call_site: Span, mode: DocCommentDesugarMode, on_enter: OnEvent, -) -> tt::TopSubtree<SpanData<Ctx>> +) -> tt::TopSubtree where - SpanMap: SpanMapper<SpanData<Ctx>>, - SpanData<Ctx>: Copy + fmt::Debug, - OnEvent: FnMut( - &mut PreorderWithTokens, - &WalkEvent<SyntaxElement>, - ) -> (bool, Vec<tt::Leaf<SpanData<Ctx>>>), + SpanMap: SpanMapper, + OnEvent: FnMut(&mut PreorderWithTokens, &WalkEvent<SyntaxElement>) -> (bool, Vec<tt::Leaf>), { let mut c = Converter::new(node, map, append, remove, call_site, mode, on_enter); convert_tokens(&mut c) @@ -152,13 +144,13 @@ where /// Converts a [`tt::Subtree`] back to a [`SyntaxNode`]. /// The produced `SpanMap` contains a mapping from the syntax nodes offsets to the subtree's spans. -pub fn token_tree_to_syntax_node<Ctx>( - tt: &tt::TopSubtree<SpanData<Ctx>>, +pub fn token_tree_to_syntax_node( + tt: &tt::TopSubtree, entry_point: parser::TopEntryPoint, - span_to_edition: &mut dyn FnMut(Ctx) -> Edition, -) -> (Parse<SyntaxNode>, SpanMap<Ctx>) + span_to_edition: &mut dyn FnMut(SyntaxContext) -> Edition, +) -> (Parse<SyntaxNode>, SpanMap) where - Ctx: Copy + fmt::Debug + PartialEq + PartialEq + Eq + Hash, + SyntaxContext: Copy + fmt::Debug + PartialEq + PartialEq + Eq + Hash, { let buffer = tt.view().strip_invisible(); let parser_input = to_parser_input(buffer, span_to_edition); @@ -183,16 +175,12 @@ where /// Convert a string to a `TokenTree`. The spans of the subtree will be anchored to the provided /// anchor with the given context. -pub fn parse_to_token_tree<Ctx>( +pub fn parse_to_token_tree( edition: Edition, anchor: SpanAnchor, - ctx: Ctx, + ctx: SyntaxContext, text: &str, -) -> Option<tt::TopSubtree<SpanData<Ctx>>> -where - SpanData<Ctx>: Copy + fmt::Debug, - Ctx: Copy, -{ +) -> Option<tt::TopSubtree> { let lexed = parser::LexedStr::new(edition, text); if lexed.errors().next().is_some() { return None; @@ -203,14 +191,11 @@ where } /// Convert a string to a `TokenTree`. The passed span will be used for all spans of the produced subtree. -pub fn parse_to_token_tree_static_span<S>( +pub fn parse_to_token_tree_static_span( edition: Edition, - span: S, + span: Span, text: &str, -) -> Option<tt::TopSubtree<S>> -where - S: Copy + fmt::Debug, -{ +) -> Option<tt::TopSubtree> { let lexed = parser::LexedStr::new(edition, text); if lexed.errors().next().is_some() { return None; @@ -220,10 +205,9 @@ where Some(convert_tokens(&mut conv)) } -fn convert_tokens<S, C>(conv: &mut C) -> tt::TopSubtree<S> +fn convert_tokens<C>(conv: &mut C) -> tt::TopSubtree where - C: TokenConverter<S>, - S: Copy + fmt::Debug, + C: TokenConverter, C::Token: fmt::Debug, { let mut builder = @@ -239,7 +223,7 @@ where spacing: _, })) => { let found_expected_delimiter = - builder.expected_delimiters().enumerate().find(|(_, delim)| match delim.kind { + builder.expected_delimiters().enumerate().find(|(_, delim)| match delim { tt::DelimiterKind::Parenthesis => char == ')', tt::DelimiterKind::Brace => char == '}', tt::DelimiterKind::Bracket => char == ']', @@ -273,13 +257,11 @@ where } kind if kind.is_punct() && kind != UNDERSCORE => { let found_expected_delimiter = - builder.expected_delimiters().enumerate().find(|(_, delim)| { - match delim.kind { - tt::DelimiterKind::Parenthesis => kind == T![')'], - tt::DelimiterKind::Brace => kind == T!['}'], - tt::DelimiterKind::Bracket => kind == T![']'], - tt::DelimiterKind::Invisible => false, - } + builder.expected_delimiters().enumerate().find(|(_, delim)| match delim { + tt::DelimiterKind::Parenthesis => kind == T![')'], + tt::DelimiterKind::Brace => kind == T!['}'], + tt::DelimiterKind::Bracket => kind == T![']'], + tt::DelimiterKind::Invisible => false, }); // Current token is a closing delimiter that we expect, fix up the closing span @@ -327,7 +309,7 @@ where .into() }; } - let leaf: tt::Leaf<_> = match kind { + let leaf: tt::Leaf = match kind { k if k.is_any_identifier() => { let text = token.to_text(conv); tt::Ident::new(&text, conv.span_for(abs_range)).into() @@ -435,11 +417,11 @@ pub fn desugar_doc_comment_text(text: &str, mode: DocCommentDesugarMode) -> (Sym } } -fn convert_doc_comment<S: Copy>( +fn convert_doc_comment( token: &syntax::SyntaxToken, - span: S, + span: Span, mode: DocCommentDesugarMode, - builder: &mut tt::TopSubtreeBuilder<S>, + builder: &mut tt::TopSubtreeBuilder, ) { let Some(comment) = ast::Comment::cast(token.clone()) else { return }; let Some(doc) = comment.kind().doc else { return }; @@ -460,7 +442,7 @@ fn convert_doc_comment<S: Copy>( text = &text[0..text.len() - 2]; } let (text, kind) = desugar_doc_comment_text(text, mode); - let lit = tt::Literal { symbol: text, span, kind, suffix: None }; + let lit = tt::Literal { text_and_suffix: text, span, kind, suffix_len: 0 }; tt::Leaf::from(lit) }; @@ -479,92 +461,84 @@ fn convert_doc_comment<S: Copy>( } /// A raw token (straight from lexer) converter -struct RawConverter<'a, Ctx> { +struct RawConverter<'a> { lexed: parser::LexedStr<'a>, pos: usize, anchor: SpanAnchor, - ctx: Ctx, + ctx: SyntaxContext, mode: DocCommentDesugarMode, } /// A raw token (straight from lexer) converter that gives every token the same span. -struct StaticRawConverter<'a, S> { +struct StaticRawConverter<'a> { lexed: parser::LexedStr<'a>, pos: usize, - span: S, + span: Span, mode: DocCommentDesugarMode, } -trait SrcToken<Ctx, S> { +trait SrcToken<Ctx> { fn kind(&self, ctx: &Ctx) -> SyntaxKind; fn to_char(&self, ctx: &Ctx) -> Option<char>; fn to_text(&self, ctx: &Ctx) -> SmolStr; - fn as_leaf(&self) -> Option<&tt::Leaf<S>> { + fn as_leaf(&self) -> Option<&tt::Leaf> { None } } -trait TokenConverter<S>: Sized { - type Token: SrcToken<Self, S>; +trait TokenConverter: Sized { + type Token: SrcToken<Self>; fn convert_doc_comment( &self, token: &Self::Token, - span: S, - builder: &mut tt::TopSubtreeBuilder<S>, + span: Span, + builder: &mut tt::TopSubtreeBuilder, ); fn bump(&mut self) -> Option<(Self::Token, TextRange)>; fn peek(&self) -> Option<Self::Token>; - fn span_for(&self, range: TextRange) -> S; + fn span_for(&self, range: TextRange) -> Span; - fn call_site(&self) -> S; + fn call_site(&self) -> Span; } -impl<S, Ctx> SrcToken<RawConverter<'_, Ctx>, S> for usize { - fn kind(&self, ctx: &RawConverter<'_, Ctx>) -> SyntaxKind { +impl SrcToken<RawConverter<'_>> for usize { + fn kind(&self, ctx: &RawConverter<'_>) -> SyntaxKind { ctx.lexed.kind(*self) } - fn to_char(&self, ctx: &RawConverter<'_, Ctx>) -> Option<char> { + fn to_char(&self, ctx: &RawConverter<'_>) -> Option<char> { ctx.lexed.text(*self).chars().next() } - fn to_text(&self, ctx: &RawConverter<'_, Ctx>) -> SmolStr { + fn to_text(&self, ctx: &RawConverter<'_>) -> SmolStr { ctx.lexed.text(*self).into() } } -impl<S: Copy> SrcToken<StaticRawConverter<'_, S>, S> for usize { - fn kind(&self, ctx: &StaticRawConverter<'_, S>) -> SyntaxKind { +impl SrcToken<StaticRawConverter<'_>> for usize { + fn kind(&self, ctx: &StaticRawConverter<'_>) -> SyntaxKind { ctx.lexed.kind(*self) } - fn to_char(&self, ctx: &StaticRawConverter<'_, S>) -> Option<char> { + fn to_char(&self, ctx: &StaticRawConverter<'_>) -> Option<char> { ctx.lexed.text(*self).chars().next() } - fn to_text(&self, ctx: &StaticRawConverter<'_, S>) -> SmolStr { + fn to_text(&self, ctx: &StaticRawConverter<'_>) -> SmolStr { ctx.lexed.text(*self).into() } } -impl<Ctx: Copy> TokenConverter<SpanData<Ctx>> for RawConverter<'_, Ctx> -where - SpanData<Ctx>: Copy, -{ +impl TokenConverter for RawConverter<'_> { type Token = usize; - fn convert_doc_comment( - &self, - &token: &usize, - span: SpanData<Ctx>, - builder: &mut tt::TopSubtreeBuilder<SpanData<Ctx>>, - ) { + fn convert_doc_comment(&self, &token: &usize, span: Span, builder: &mut tt::TopSubtreeBuilder) { let text = self.lexed.text(token); convert_doc_comment(&doc_comment(text), span, self.mode, builder); } @@ -588,22 +562,19 @@ where Some(self.pos) } - fn span_for(&self, range: TextRange) -> SpanData<Ctx> { - SpanData { range, anchor: self.anchor, ctx: self.ctx } + fn span_for(&self, range: TextRange) -> Span { + Span { range, anchor: self.anchor, ctx: self.ctx } } - fn call_site(&self) -> SpanData<Ctx> { - SpanData { range: TextRange::empty(0.into()), anchor: self.anchor, ctx: self.ctx } + fn call_site(&self) -> Span { + Span { range: TextRange::empty(0.into()), anchor: self.anchor, ctx: self.ctx } } } -impl<S> TokenConverter<S> for StaticRawConverter<'_, S> -where - S: Copy, -{ +impl TokenConverter for StaticRawConverter<'_> { type Token = usize; - fn convert_doc_comment(&self, &token: &usize, span: S, builder: &mut tt::TopSubtreeBuilder<S>) { + fn convert_doc_comment(&self, &token: &usize, span: Span, builder: &mut tt::TopSubtreeBuilder) { let text = self.lexed.text(token); convert_doc_comment(&doc_comment(text), span, self.mode, builder); } @@ -627,40 +598,40 @@ where Some(self.pos) } - fn span_for(&self, _: TextRange) -> S { + fn span_for(&self, _: TextRange) -> Span { self.span } - fn call_site(&self) -> S { + fn call_site(&self) -> Span { self.span } } -struct Converter<SpanMap, S, OnEvent> { +struct Converter<SpanMap, OnEvent> { current: Option<SyntaxToken>, - current_leaves: VecDeque<tt::Leaf<S>>, + current_leaves: VecDeque<tt::Leaf>, preorder: PreorderWithTokens, range: TextRange, punct_offset: Option<(SyntaxToken, TextSize)>, /// Used to make the emitted text ranges in the spans relative to the span anchor. map: SpanMap, - append: FxHashMap<SyntaxElement, Vec<tt::Leaf<S>>>, + append: FxHashMap<SyntaxElement, Vec<tt::Leaf>>, remove: FxHashSet<SyntaxElement>, - call_site: S, + call_site: Span, mode: DocCommentDesugarMode, on_event: OnEvent, } -impl<SpanMap, S, OnEvent> Converter<SpanMap, S, OnEvent> +impl<SpanMap, OnEvent> Converter<SpanMap, OnEvent> where - OnEvent: FnMut(&mut PreorderWithTokens, &WalkEvent<SyntaxElement>) -> (bool, Vec<tt::Leaf<S>>), + OnEvent: FnMut(&mut PreorderWithTokens, &WalkEvent<SyntaxElement>) -> (bool, Vec<tt::Leaf>), { fn new( node: &SyntaxNode, map: SpanMap, - append: FxHashMap<SyntaxElement, Vec<tt::Leaf<S>>>, + append: FxHashMap<SyntaxElement, Vec<tt::Leaf>>, remove: FxHashSet<SyntaxElement>, - call_site: S, + call_site: Span, mode: DocCommentDesugarMode, on_enter: OnEvent, ) -> Self { @@ -720,13 +691,13 @@ where } #[derive(Debug)] -enum SynToken<S> { +enum SynToken { Ordinary(SyntaxToken), Punct { token: SyntaxToken, offset: usize }, - Leaf(tt::Leaf<S>), + Leaf(tt::Leaf), } -impl<S> SynToken<S> { +impl SynToken { fn token(&self) -> &SyntaxToken { match self { SynToken::Ordinary(it) | SynToken::Punct { token: it, offset: _ } => it, @@ -735,8 +706,8 @@ impl<S> SynToken<S> { } } -impl<SpanMap, S, OnEvent> SrcToken<Converter<SpanMap, S, OnEvent>, S> for SynToken<S> { - fn kind(&self, _ctx: &Converter<SpanMap, S, OnEvent>) -> SyntaxKind { +impl<SpanMap, OnEvent> SrcToken<Converter<SpanMap, OnEvent>> for SynToken { + fn kind(&self, _ctx: &Converter<SpanMap, OnEvent>) -> SyntaxKind { match self { SynToken::Ordinary(token) => token.kind(), SynToken::Punct { token, offset: i } => { @@ -748,14 +719,14 @@ impl<SpanMap, S, OnEvent> SrcToken<Converter<SpanMap, S, OnEvent>, S> for SynTok } } } - fn to_char(&self, _ctx: &Converter<SpanMap, S, OnEvent>) -> Option<char> { + fn to_char(&self, _ctx: &Converter<SpanMap, OnEvent>) -> Option<char> { match self { SynToken::Ordinary(_) => None, SynToken::Punct { token: it, offset: i } => it.text().chars().nth(*i), SynToken::Leaf(_) => None, } } - fn to_text(&self, _ctx: &Converter<SpanMap, S, OnEvent>) -> SmolStr { + fn to_text(&self, _ctx: &Converter<SpanMap, OnEvent>) -> SmolStr { match self { SynToken::Ordinary(token) | SynToken::Punct { token, offset: _ } => token.text().into(), SynToken::Leaf(_) => { @@ -764,7 +735,7 @@ impl<SpanMap, S, OnEvent> SrcToken<Converter<SpanMap, S, OnEvent>, S> for SynTok } } } - fn as_leaf(&self) -> Option<&tt::Leaf<S>> { + fn as_leaf(&self) -> Option<&tt::Leaf> { match self { SynToken::Ordinary(_) | SynToken::Punct { .. } => None, SynToken::Leaf(it) => Some(it), @@ -772,18 +743,17 @@ impl<SpanMap, S, OnEvent> SrcToken<Converter<SpanMap, S, OnEvent>, S> for SynTok } } -impl<S, SpanMap, OnEvent> TokenConverter<S> for Converter<SpanMap, S, OnEvent> +impl<SpanMap, OnEvent> TokenConverter for Converter<SpanMap, OnEvent> where - S: Copy, - SpanMap: SpanMapper<S>, - OnEvent: FnMut(&mut PreorderWithTokens, &WalkEvent<SyntaxElement>) -> (bool, Vec<tt::Leaf<S>>), + SpanMap: SpanMapper, + OnEvent: FnMut(&mut PreorderWithTokens, &WalkEvent<SyntaxElement>) -> (bool, Vec<tt::Leaf>), { - type Token = SynToken<S>; + type Token = SynToken; fn convert_doc_comment( &self, token: &Self::Token, - span: S, - builder: &mut tt::TopSubtreeBuilder<S>, + span: Span, + builder: &mut tt::TopSubtreeBuilder, ) { convert_doc_comment(token.token(), span, self.mode, builder); } @@ -847,30 +817,24 @@ where Some(token) } - fn span_for(&self, range: TextRange) -> S { + fn span_for(&self, range: TextRange) -> Span { self.map.span_for(range) } - fn call_site(&self) -> S { + fn call_site(&self) -> Span { self.call_site } } -struct TtTreeSink<'a, Ctx> -where - SpanData<Ctx>: Copy, -{ +struct TtTreeSink<'a> { buf: String, - cursor: Cursor<'a, SpanData<Ctx>>, + cursor: Cursor<'a>, text_pos: TextSize, inner: SyntaxTreeBuilder, - token_map: SpanMap<Ctx>, + token_map: SpanMap, } -impl<'a, Ctx> TtTreeSink<'a, Ctx> -where - SpanData<Ctx>: Copy, -{ - fn new(cursor: Cursor<'a, SpanData<Ctx>>) -> Self { +impl<'a> TtTreeSink<'a> { + fn new(cursor: Cursor<'a>) -> Self { TtTreeSink { buf: String::new(), cursor, @@ -880,7 +844,7 @@ where } } - fn finish(mut self) -> (Parse<SyntaxNode>, SpanMap<Ctx>) { + fn finish(mut self) -> (Parse<SyntaxNode>, SpanMap) { self.token_map.finish(); (self.inner.finish(), self.token_map) } @@ -898,21 +862,15 @@ fn delim_to_str(d: tt::DelimiterKind, closing: bool) -> Option<&'static str> { Some(&texts[idx..texts.len() - (1 - idx)]) } -impl<Ctx> TtTreeSink<'_, Ctx> -where - SpanData<Ctx>: Copy + fmt::Debug, - Ctx: PartialEq, -{ +impl TtTreeSink<'_> { /// Parses a float literal as if it was a one to two name ref nodes with a dot inbetween. /// This occurs when a float literal is used as a field access. fn float_split(&mut self, has_pseudo_dot: bool) { - let (text, span) = match self.cursor.token_tree() { - Some(tt::TokenTree::Leaf(tt::Leaf::Literal(tt::Literal { - symbol: text, - span, - kind: tt::LitKind::Float, - suffix: _, - }))) => (text.as_str(), *span), + let token_tree = self.cursor.token_tree(); + let (text, span) = match &token_tree { + Some(tt::TokenTree::Leaf(tt::Leaf::Literal( + lit @ tt::Literal { span, kind: tt::LitKind::Float, .. }, + ))) => (lit.text(), *span), tt => unreachable!("{tt:?}"), }; // FIXME: Span splitting @@ -971,9 +929,15 @@ where self.buf.push_str("r#"); self.text_pos += TextSize::of("r#"); } - let r = (ident.sym.as_str(), ident.span); + let text = ident.sym.as_str(); + self.buf += text; + self.text_pos += TextSize::of(text); + combined_span = match combined_span { + None => Some(ident.span), + Some(prev_span) => Some(Self::merge_spans(prev_span, ident.span)), + }; self.cursor.bump(); - r + continue 'tokens; } tt::Leaf::Punct(punct) => { assert!(punct.char.is_ascii()); @@ -1053,10 +1017,10 @@ where self.inner.error(error, self.text_pos) } - fn merge_spans(a: SpanData<Ctx>, b: SpanData<Ctx>) -> SpanData<Ctx> { + fn merge_spans(a: Span, b: Span) -> Span { // We don't do what rustc does exactly, rustc does something clever when the spans have different syntax contexts // but this runs afoul of our separation between `span` and `hir-expand`. - SpanData { + Span { range: if a.ctx == b.ctx && a.anchor == b.anchor { TextRange::new( std::cmp::min(a.range.start(), b.range.start()), |