Unnamed repository; edit this file 'description' to name the repository.
Merge #11118
11118: internal: move ws attachment logic to the parser crate r=matklad a=matklad
This has to re-introduce the `sink` pattern, because doing this purely
with iterators is awkward :( Maaaybe the event vector was a false start?
But, anyway, I like the current factoring more -- it sort-of obvious
that we do want to keep ws-attachment business in the parser, and that
we also don't want that to depend on the particular tree structure. I
think `shortcuts` module achieves that.
bors r+
🤖
Co-authored-by: Aleksey Kladov <[email protected]>
| -rw-r--r-- | crates/parser/src/lexed_str.rs | 25 | ||||
| -rw-r--r-- | crates/parser/src/lib.rs | 2 | ||||
| -rw-r--r-- | crates/parser/src/shortcuts.rs | 220 | ||||
| -rw-r--r-- | crates/syntax/src/parsing.rs | 36 | ||||
| -rw-r--r-- | crates/syntax/src/parsing/reparsing.rs | 2 | ||||
| -rw-r--r-- | crates/syntax/src/parsing/text_tree_sink.rs | 191 |
6 files changed, 255 insertions, 221 deletions
diff --git a/crates/parser/src/lexed_str.rs b/crates/parser/src/lexed_str.rs index f17aae1d31..5a42c53c94 100644 --- a/crates/parser/src/lexed_str.rs +++ b/crates/parser/src/lexed_str.rs @@ -122,31 +122,6 @@ impl<'a> LexedStr<'a> { self.error.iter().map(|it| (it.token as usize, it.msg.as_str())) } - pub fn to_input(&self) -> crate::Input { - let mut res = crate::Input::default(); - let mut was_joint = false; - for i in 0..self.len() { - let kind = self.kind(i); - if kind.is_trivia() { - was_joint = false - } else { - if kind == SyntaxKind::IDENT { - let token_text = self.text(i); - let contextual_kw = SyntaxKind::from_contextual_keyword(token_text) - .unwrap_or(SyntaxKind::IDENT); - res.push_ident(contextual_kw); - } else { - if was_joint { - res.was_joint(); - } - res.push(kind); - } - was_joint = true; - } - } - res - } - fn push(&mut self, kind: SyntaxKind, offset: usize) { self.kind.push(kind); self.start.push(offset as u32); diff --git a/crates/parser/src/lib.rs b/crates/parser/src/lib.rs index da78889f35..841d2aa4e9 100644 --- a/crates/parser/src/lib.rs +++ b/crates/parser/src/lib.rs @@ -26,6 +26,7 @@ mod parser; mod grammar; mod input; mod output; +mod shortcuts; #[cfg(test)] mod tests; @@ -36,6 +37,7 @@ pub use crate::{ input::Input, lexed_str::LexedStr, output::{Output, Step}, + shortcuts::StrStep, syntax_kind::SyntaxKind, }; diff --git a/crates/parser/src/shortcuts.rs b/crates/parser/src/shortcuts.rs new file mode 100644 index 0000000000..15387a85cc --- /dev/null +++ b/crates/parser/src/shortcuts.rs @@ -0,0 +1,220 @@ +//! Shortcuts that span lexer/parser abstraction. +//! +//! The way Rust works, parser doesn't necessary parse text, and you might +//! tokenize text without parsing it further. So, it makes sense to keep +//! abstract token parsing, and string tokenization as completely separate +//! layers. +//! +//! However, often you do pares text into syntax trees and the glue code for +//! that needs to live somewhere. Rather than putting it to lexer or parser, we +//! use a separate shortcuts module for that. + +use std::mem; + +use crate::{ + LexedStr, Step, + SyntaxKind::{self, *}, +}; + +pub enum StrStep<'a> { + Token { kind: SyntaxKind, text: &'a str }, + Enter { kind: SyntaxKind }, + Exit, + Error { msg: &'a str, pos: usize }, +} + +impl<'a> LexedStr<'a> { + pub fn to_input(&self) -> crate::Input { + let mut res = crate::Input::default(); + let mut was_joint = false; + for i in 0..self.len() { + let kind = self.kind(i); + if kind.is_trivia() { + was_joint = false + } else { + if kind == SyntaxKind::IDENT { + let token_text = self.text(i); + let contextual_kw = SyntaxKind::from_contextual_keyword(token_text) + .unwrap_or(SyntaxKind::IDENT); + res.push_ident(contextual_kw); + } else { + if was_joint { + res.was_joint(); + } + res.push(kind); + } + was_joint = true; + } + } + res + } + + pub fn intersperse_trivia( + &self, + output: &crate::Output, + synthetic_root: bool, + sink: &mut dyn FnMut(StrStep), + ) -> bool { + let mut builder = Builder { lexed: self, pos: 0, state: State::PendingEnter, sink }; + + if synthetic_root { + builder.enter(SyntaxKind::SOURCE_FILE); + } + for event in output.iter() { + match event { + Step::Token { kind, n_input_tokens: n_raw_tokens } => { + builder.token(kind, n_raw_tokens) + } + Step::Enter { kind } => builder.enter(kind), + Step::Exit => builder.exit(), + Step::Error { msg } => { + let text_pos = builder.lexed.text_start(builder.pos); + (builder.sink)(StrStep::Error { msg, pos: text_pos }); + } + } + } + if synthetic_root { + builder.exit(); + } + + match mem::replace(&mut builder.state, State::Normal) { + State::PendingExit => { + builder.eat_trivias(); + (builder.sink)(StrStep::Exit); + } + State::PendingEnter | State::Normal => unreachable!(), + } + + let is_eof = builder.pos == builder.lexed.len(); + is_eof + } +} + +struct Builder<'a, 'b> { + lexed: &'a LexedStr<'a>, + pos: usize, + state: State, + sink: &'b mut dyn FnMut(StrStep<'_>), +} + +enum State { + PendingEnter, + Normal, + PendingExit, +} + +impl Builder<'_, '_> { + fn token(&mut self, kind: SyntaxKind, n_tokens: u8) { + match mem::replace(&mut self.state, State::Normal) { + State::PendingEnter => unreachable!(), + State::PendingExit => (self.sink)(StrStep::Exit), + State::Normal => (), + } + self.eat_trivias(); + self.do_token(kind, n_tokens as usize); + } + + fn enter(&mut self, kind: SyntaxKind) { + match mem::replace(&mut self.state, State::Normal) { + State::PendingEnter => { + (self.sink)(StrStep::Enter { kind }); + // No need to attach trivias to previous node: there is no + // previous node. + return; + } + State::PendingExit => (self.sink)(StrStep::Exit), + State::Normal => (), + } + + let n_trivias = + (self.pos..self.lexed.len()).take_while(|&it| self.lexed.kind(it).is_trivia()).count(); + let leading_trivias = self.pos..self.pos + n_trivias; + let n_attached_trivias = n_attached_trivias( + kind, + leading_trivias.rev().map(|it| (self.lexed.kind(it), self.lexed.text(it))), + ); + self.eat_n_trivias(n_trivias - n_attached_trivias); + (self.sink)(StrStep::Enter { kind }); + self.eat_n_trivias(n_attached_trivias); + } + + fn exit(&mut self) { + match mem::replace(&mut self.state, State::PendingExit) { + State::PendingEnter => unreachable!(), + State::PendingExit => (self.sink)(StrStep::Exit), + State::Normal => (), + } + } + + fn eat_trivias(&mut self) { + while self.pos < self.lexed.len() { + let kind = self.lexed.kind(self.pos); + if !kind.is_trivia() { + break; + } + self.do_token(kind, 1); + } + } + + fn eat_n_trivias(&mut self, n: usize) { + for _ in 0..n { + let kind = self.lexed.kind(self.pos); + assert!(kind.is_trivia()); + self.do_token(kind, 1); + } + } + + fn do_token(&mut self, kind: SyntaxKind, n_tokens: usize) { + let text = &self.lexed.range_text(self.pos..self.pos + n_tokens); + self.pos += n_tokens; + (self.sink)(StrStep::Token { kind, text }); + } +} + +fn n_attached_trivias<'a>( + kind: SyntaxKind, + trivias: impl Iterator<Item = (SyntaxKind, &'a str)>, +) -> usize { + match kind { + CONST | ENUM | FN | IMPL | MACRO_CALL | MACRO_DEF | MACRO_RULES | MODULE | RECORD_FIELD + | STATIC | STRUCT | TRAIT | TUPLE_FIELD | TYPE_ALIAS | UNION | USE | VARIANT => { + let mut res = 0; + let mut trivias = trivias.enumerate().peekable(); + + while let Some((i, (kind, text))) = trivias.next() { + match kind { + WHITESPACE if text.contains("\n\n") => { + // we check whether the next token is a doc-comment + // and skip the whitespace in this case + if let Some((COMMENT, peek_text)) = trivias.peek().map(|(_, pair)| pair) { + if is_outer(peek_text) { + continue; + } + } + break; + } + COMMENT => { + if is_inner(text) { + break; + } + res = i + 1; + } + _ => (), + } + } + res + } + _ => 0, + } +} + +fn is_outer(text: &str) -> bool { + if text.starts_with("////") || text.starts_with("/***") { + return false; + } + text.starts_with("///") || text.starts_with("/**") +} + +fn is_inner(text: &str) -> bool { + text.starts_with("//!") || text.starts_with("/*!") +} diff --git a/crates/syntax/src/parsing.rs b/crates/syntax/src/parsing.rs index 20c7101a03..971fa2700d 100644 --- a/crates/syntax/src/parsing.rs +++ b/crates/syntax/src/parsing.rs @@ -1,12 +1,11 @@ //! Lexing, bridging to parser (which does the actual parsing) and //! incremental reparsing. -mod text_tree_sink; mod reparsing; -use crate::{ - parsing::text_tree_sink::build_tree, syntax_node::GreenNode, AstNode, SyntaxError, SyntaxNode, -}; +use rowan::TextRange; + +use crate::{syntax_node::GreenNode, AstNode, SyntaxError, SyntaxNode, SyntaxTreeBuilder}; pub(crate) use crate::parsing::reparsing::incremental_reparse; @@ -37,3 +36,32 @@ pub(crate) fn parse_text_as<T: AstNode>( SyntaxNode::new_root(node).first_child().and_then(T::cast).ok_or(()) } + +pub(crate) fn build_tree( + lexed: parser::LexedStr<'_>, + parser_output: parser::Output, + synthetic_root: bool, +) -> (GreenNode, Vec<SyntaxError>, bool) { + let mut builder = SyntaxTreeBuilder::default(); + + let is_eof = lexed.intersperse_trivia(&parser_output, synthetic_root, &mut |step| match step { + parser::StrStep::Token { kind, text } => builder.token(kind, text), + parser::StrStep::Enter { kind } => builder.start_node(kind), + parser::StrStep::Exit => builder.finish_node(), + parser::StrStep::Error { msg, pos } => { + builder.error(msg.to_string(), pos.try_into().unwrap()) + } + }); + + let (node, mut errors) = builder.finish_raw(); + for (i, err) in lexed.errors() { + let text_range = lexed.text_range(i); + let text_range = TextRange::new( + text_range.start.try_into().unwrap(), + text_range.end.try_into().unwrap(), + ); + errors.push(SyntaxError::new(err, text_range)) + } + + (node, errors, is_eof) +} diff --git a/crates/syntax/src/parsing/reparsing.rs b/crates/syntax/src/parsing/reparsing.rs index a6abe3cccf..dd2e01bfc4 100644 --- a/crates/syntax/src/parsing/reparsing.rs +++ b/crates/syntax/src/parsing/reparsing.rs @@ -10,7 +10,7 @@ use parser::Reparser; use text_edit::Indel; use crate::{ - parsing::text_tree_sink::build_tree, + parsing::build_tree, syntax_node::{GreenNode, GreenToken, NodeOrToken, SyntaxElement, SyntaxNode}, SyntaxError, SyntaxKind::*, diff --git a/crates/syntax/src/parsing/text_tree_sink.rs b/crates/syntax/src/parsing/text_tree_sink.rs deleted file mode 100644 index f40c549e3d..0000000000 --- a/crates/syntax/src/parsing/text_tree_sink.rs +++ /dev/null @@ -1,191 +0,0 @@ -//! See [`TextTreeSink`]. - -use std::mem; - -use parser::LexedStr; - -use crate::{ - ast, - syntax_node::GreenNode, - SyntaxError, - SyntaxKind::{self, *}, - SyntaxTreeBuilder, TextRange, -}; - -pub(crate) fn build_tree( - lexed: LexedStr<'_>, - parser_output: parser::Output, - synthetic_root: bool, -) -> (GreenNode, Vec<SyntaxError>, bool) { - let mut builder = Builder::new(lexed); - - if synthetic_root { - builder.enter(SyntaxKind::SOURCE_FILE); - } - - for event in parser_output.iter() { - match event { - parser::Step::Token { kind, n_input_tokens: n_raw_tokens } => { - builder.token(kind, n_raw_tokens) - } - parser::Step::Enter { kind } => builder.enter(kind), - parser::Step::Exit => builder.exit(), - parser::Step::Error { msg } => { - let text_pos = builder.lexed.text_start(builder.pos).try_into().unwrap(); - builder.inner.error(msg.to_string(), text_pos); - } - } - } - if synthetic_root { - builder.exit() - } - builder.build() -} - -struct Builder<'a> { - lexed: LexedStr<'a>, - pos: usize, - state: State, - inner: SyntaxTreeBuilder, -} - -enum State { - PendingStart, - Normal, - PendingFinish, -} - -impl<'a> Builder<'a> { - fn new(lexed: parser::LexedStr<'a>) -> Self { - Self { lexed, pos: 0, state: State::PendingStart, inner: SyntaxTreeBuilder::default() } - } - - fn build(mut self) -> (GreenNode, Vec<SyntaxError>, bool) { - match mem::replace(&mut self.state, State::Normal) { - State::PendingFinish => { - self.eat_trivias(); - self.inner.finish_node(); - } - State::PendingStart | State::Normal => unreachable!(), - } - - let (node, mut errors) = self.inner.finish_raw(); - for (i, err) in self.lexed.errors() { - let text_range = self.lexed.text_range(i); - let text_range = TextRange::new( - text_range.start.try_into().unwrap(), - text_range.end.try_into().unwrap(), - ); - errors.push(SyntaxError::new(err, text_range)) - } - - let is_eof = self.pos == self.lexed.len(); - - (node, errors, is_eof) - } - - fn token(&mut self, kind: SyntaxKind, n_tokens: u8) { - match mem::replace(&mut self.state, State::Normal) { - State::PendingStart => unreachable!(), - State::PendingFinish => self.inner.finish_node(), - State::Normal => (), - } - self.eat_trivias(); - self.do_token(kind, n_tokens as usize); - } - - fn enter(&mut self, kind: SyntaxKind) { - match mem::replace(&mut self.state, State::Normal) { - State::PendingStart => { - self.inner.start_node(kind); - // No need to attach trivias to previous node: there is no - // previous node. - return; - } - State::PendingFinish => self.inner.finish_node(), - State::Normal => (), - } - - let n_trivias = - (self.pos..self.lexed.len()).take_while(|&it| self.lexed.kind(it).is_trivia()).count(); - let leading_trivias = self.pos..self.pos + n_trivias; - let n_attached_trivias = n_attached_trivias( - kind, - leading_trivias.rev().map(|it| (self.lexed.kind(it), self.lexed.text(it))), - ); - self.eat_n_trivias(n_trivias - n_attached_trivias); - self.inner.start_node(kind); - self.eat_n_trivias(n_attached_trivias); - } - - fn exit(&mut self) { - match mem::replace(&mut self.state, State::PendingFinish) { - State::PendingStart => unreachable!(), - State::PendingFinish => self.inner.finish_node(), - State::Normal => (), - } - } - - fn eat_trivias(&mut self) { - while self.pos < self.lexed.len() { - let kind = self.lexed.kind(self.pos); - if !kind.is_trivia() { - break; - } - self.do_token(kind, 1); - } - } - - fn eat_n_trivias(&mut self, n: usize) { - for _ in 0..n { - let kind = self.lexed.kind(self.pos); - assert!(kind.is_trivia()); - self.do_token(kind, 1); - } - } - - fn do_token(&mut self, kind: SyntaxKind, n_tokens: usize) { - let text = &self.lexed.range_text(self.pos..self.pos + n_tokens); - self.pos += n_tokens; - self.inner.token(kind, text); - } -} - -fn n_attached_trivias<'a>( - kind: SyntaxKind, - trivias: impl Iterator<Item = (SyntaxKind, &'a str)>, -) -> usize { - match kind { - CONST | ENUM | FN | IMPL | MACRO_CALL | MACRO_DEF | MACRO_RULES | MODULE | RECORD_FIELD - | STATIC | STRUCT | TRAIT | TUPLE_FIELD | TYPE_ALIAS | UNION | USE | VARIANT => { - let mut res = 0; - let mut trivias = trivias.enumerate().peekable(); - - while let Some((i, (kind, text))) = trivias.next() { - match kind { - WHITESPACE if text.contains("\n\n") => { - // we check whether the next token is a doc-comment - // and skip the whitespace in this case - if let Some((COMMENT, peek_text)) = trivias.peek().map(|(_, pair)| pair) { - let comment_kind = ast::CommentKind::from_text(peek_text); - if comment_kind.doc == Some(ast::CommentPlacement::Outer) { - continue; - } - } - break; - } - COMMENT => { - let comment_kind = ast::CommentKind::from_text(text); - if comment_kind.doc == Some(ast::CommentPlacement::Inner) { - break; - } - res = i + 1; - } - _ => (), - } - } - res - } - _ => 0, - } -} |