rust-analyzer

diff --git a/crates/mbe/src/lib.rs b/crates/mbe/src/lib.rs
index 1a56878fdb..5e14a3fb59 100644
--- a/crates/mbe/src/lib.rs
+++ b/crates/mbe/src/lib.rs

@@ -10,7 +10,7 @@ mod parser;

mod expander;

mod syntax_bridge;

mod tt_iter;

-mod to_parser_tokens;

+mod to_parser_input;

#[cfg(test)]

mod benchmark;

diff --git a/crates/mbe/src/syntax_bridge.rs b/crates/mbe/src/syntax_bridge.rs
index 109842b0cd..f0c1f806ff 100644
--- a/crates/mbe/src/syntax_bridge.rs
+++ b/crates/mbe/src/syntax_bridge.rs

@@ -1,6 +1,5 @@

//! Conversions between [`SyntaxNode`] and [`tt::TokenTree`].

-use parser::{ParseError, TreeSink};

use rustc_hash::{FxHashMap, FxHashSet};

use syntax::{

ast::{self, make::tokens::doc_comment},

@@ -11,7 +10,7 @@ use syntax::{

use tt::buffer::{Cursor, TokenBuffer};

use crate::{

- to_parser_tokens::to_parser_tokens, tt_iter::TtIter, ExpandError, ParserEntryPoint, TokenMap,

+ to_parser_input::to_parser_input, tt_iter::TtIter, ExpandError, ParserEntryPoint, TokenMap,

};

/// Convert the syntax node to a `TokenTree` (what macro

@@ -55,9 +54,19 @@ pub fn token_tree_to_syntax_node(

}

_ => TokenBuffer::from_subtree(tt),

};

- let parser_tokens = to_parser_tokens(&buffer);

+ let parser_input = to_parser_input(&buffer);

+ let parser_output = parser::parse(&parser_input, entry_point);

let mut tree_sink = TtTreeSink::new(buffer.begin());

- parser::parse(&parser_tokens, &mut tree_sink, entry_point);

+ for event in parser_output.iter() {

+ match event {

+ parser::Step::Token { kind, n_input_tokens: n_raw_tokens } => {

+ tree_sink.token(kind, n_raw_tokens)

+ }

+ parser::Step::Enter { kind } => tree_sink.start_node(kind),

+ parser::Step::Exit => tree_sink.finish_node(),

+ parser::Step::Error { msg } => tree_sink.error(msg.to_string()),

+ }

if tree_sink.roots.len() != 1 {

return Err(ExpandError::ConversionError);

}

@@ -643,7 +652,7 @@ fn delim_to_str(d: tt::DelimiterKind, closing: bool) -> &'static str {

&texts[idx..texts.len() - (1 - idx)]

}

-impl<'a> TreeSink for TtTreeSink<'a> {

+impl<'a> TtTreeSink<'a> {

fn token(&mut self, kind: SyntaxKind, mut n_tokens: u8) {

if kind == LIFETIME_IDENT {

n_tokens = 2;

@@ -741,7 +750,7 @@ impl<'a> TreeSink for TtTreeSink<'a> {

*self.roots.last_mut().unwrap() -= 1;

}

- fn error(&mut self, error: ParseError) {

+ fn error(&mut self, error: String) {

self.inner.error(error, self.text_pos)

}

diff --git a/crates/mbe/src/to_parser_tokens.rs b/crates/mbe/src/to_parser_input.rs
index f419c78d46..6faa147218 100644
--- a/crates/mbe/src/to_parser_tokens.rs
+++ b/crates/mbe/src/to_parser_input.rs

@@ -4,8 +4,8 @@

use syntax::{SyntaxKind, SyntaxKind::*, T};

use tt::buffer::TokenBuffer;

-pub(crate) fn to_parser_tokens(buffer: &TokenBuffer) -> parser::Tokens {

- let mut res = parser::Tokens::default();

+pub(crate) fn to_parser_input(buffer: &TokenBuffer) -> parser::Input {

+ let mut res = parser::Input::default();

let mut current = buffer.begin();

diff --git a/crates/mbe/src/tt_iter.rs b/crates/mbe/src/tt_iter.rs
index d05e84b0f0..2d2dbd8994 100644
--- a/crates/mbe/src/tt_iter.rs
+++ b/crates/mbe/src/tt_iter.rs

@@ -1,11 +1,10 @@

//! A "Parser" structure for token trees. We use this when parsing a declarative

//! macro definition into a list of patterns and templates.

-use crate::{to_parser_tokens::to_parser_tokens, ExpandError, ExpandResult, ParserEntryPoint};

+use crate::{to_parser_input::to_parser_input, ExpandError, ExpandResult, ParserEntryPoint};

-use parser::TreeSink;

use syntax::SyntaxKind;

-use tt::buffer::{Cursor, TokenBuffer};

+use tt::buffer::TokenBuffer;

macro_rules! err {

() => {

@@ -94,34 +93,28 @@ impl<'a> TtIter<'a> {

&mut self,

entry_point: ParserEntryPoint,

) -> ExpandResult<Option<tt::TokenTree>> {

- struct OffsetTokenSink<'a> {

- cursor: Cursor<'a>,

- error: bool,

- }

- impl<'a> TreeSink for OffsetTokenSink<'a> {

- fn token(&mut self, kind: SyntaxKind, mut n_tokens: u8) {

- if kind == SyntaxKind::LIFETIME_IDENT {

- n_tokens = 2;

- }

- for _ in 0..n_tokens {

- self.cursor = self.cursor.bump_subtree();

+ let buffer = TokenBuffer::from_tokens(self.inner.as_slice());

+ let parser_input = to_parser_input(&buffer);

+ let tree_traversal = parser::parse(&parser_input, entry_point);

+ let mut cursor = buffer.begin();

+ let mut error = false;

+ for step in tree_traversal.iter() {

+ match step {

+ parser::Step::Token { kind, mut n_input_tokens } => {

+ if kind == SyntaxKind::LIFETIME_IDENT {

+ n_input_tokens = 2;

+ }

+ for _ in 0..n_input_tokens {

+ cursor = cursor.bump_subtree();

+ }

}

- }

- fn start_node(&mut self, _kind: SyntaxKind) {}

- fn finish_node(&mut self) {}

- fn error(&mut self, _error: parser::ParseError) {

- self.error = true;

+ parser::Step::Enter { .. } | parser::Step::Exit => (),

+ parser::Step::Error { .. } => error = true,

}

- let buffer = TokenBuffer::from_tokens(self.inner.as_slice());

- let parser_tokens = to_parser_tokens(&buffer);

- let mut sink = OffsetTokenSink { cursor: buffer.begin(), error: false };

- parser::parse(&parser_tokens, &mut sink, entry_point);

- let mut err = if !sink.cursor.is_root() || sink.error {

+ let mut err = if !cursor.is_root() || error {

Some(err!("expected {:?}", entry_point))

} else {

None

@@ -130,8 +123,8 @@ impl<'a> TtIter<'a> {

let mut curr = buffer.begin();

let mut res = vec![];

- if sink.cursor.is_root() {

- while curr != sink.cursor {

+ if cursor.is_root() {

+ while curr != cursor {

if let Some(token) = curr.token_tree() {

res.push(token);

}

diff --git a/crates/parser/src/event.rs b/crates/parser/src/event.rs
index 41b0328027..b0e70e7943 100644
--- a/crates/parser/src/event.rs
+++ b/crates/parser/src/event.rs

@@ -10,9 +10,8 @@

use std::mem;

use crate::{

- ParseError,

+ output::Output,

SyntaxKind::{self, *},

- TreeSink,

};

/// `Parser` produces a flat list of `Event`s.

@@ -77,7 +76,7 @@ pub(crate) enum Event {

Error {

- msg: ParseError,

+ msg: String,

}

@@ -88,7 +87,8 @@ impl Event {

}

/// Generate the syntax tree with the control of events.

-pub(super) fn process(sink: &mut dyn TreeSink, mut events: Vec<Event>) {

+pub(super) fn process(mut events: Vec<Event>) -> Output {

+ let mut res = Output::default();

let mut forward_parents = Vec::new();

for i in 0..events.len() {

@@ -117,15 +117,17 @@ pub(super) fn process(sink: &mut dyn TreeSink, mut events: Vec<Event>) {

for kind in forward_parents.drain(..).rev() {

if kind != TOMBSTONE {

- sink.start_node(kind);

+ res.enter_node(kind);

}

- Event::Finish => sink.finish_node(),

+ Event::Finish => res.leave_node(),

Event::Token { kind, n_raw_tokens } => {

- sink.token(kind, n_raw_tokens);

+ res.token(kind, n_raw_tokens);

}

- Event::Error { msg } => sink.error(msg),

+ Event::Error { msg } => res.error(msg),

}

+ res

}

diff --git a/crates/parser/src/tokens.rs b/crates/parser/src/input.rs
index 4fc2361add..9504bd4d9e 100644
--- a/crates/parser/src/tokens.rs
+++ b/crates/parser/src/input.rs

@@ -1,26 +1,26 @@

-//! Input for the parser -- a sequence of tokens.

-//!

-//! As of now, parser doesn't have access to the *text* of the tokens, and makes

-//! decisions based solely on their classification. Unlike `LexerToken`, the

-//! `Tokens` doesn't include whitespace and comments.

+//! See [`Input`].

use crate::SyntaxKind;

#[allow(non_camel_case_types)]

type bits = u64;

-/// Main input to the parser.

+/// Input for the parser -- a sequence of tokens.

///

-/// A sequence of tokens represented internally as a struct of arrays.

+/// As of now, parser doesn't have access to the *text* of the tokens, and makes

+/// decisions based solely on their classification. Unlike `LexerToken`, the

+/// `Tokens` doesn't include whitespace and comments. Main input to the parser.

+///

+/// Struct of arrays internally, but this shouldn't really matter.

#[derive(Default)]

-pub struct Tokens {

+pub struct Input {

kind: Vec<SyntaxKind>,

joint: Vec<bits>,

contextual_kind: Vec<SyntaxKind>,

}

/// `pub` impl used by callers to create `Tokens`.

-impl Tokens {

+impl Input {

#[inline]

pub fn push(&mut self, kind: SyntaxKind) {

self.push_impl(kind, SyntaxKind::EOF)

@@ -63,7 +63,7 @@ impl Tokens {

}

/// pub(crate) impl used by the parser to consume `Tokens`.

-impl Tokens {

+impl Input {

pub(crate) fn kind(&self, idx: usize) -> SyntaxKind {

self.kind.get(idx).copied().unwrap_or(SyntaxKind::EOF)

}

@@ -76,7 +76,7 @@ impl Tokens {

}

-impl Tokens {

+impl Input {

fn bit_index(&self, n: usize) -> (usize, usize) {

let idx = n / (bits::BITS as usize);

let b_idx = n % (bits::BITS as usize);

diff --git a/crates/parser/src/lexed_str.rs b/crates/parser/src/lexed_str.rs
index b8936c3440..f17aae1d31 100644
--- a/crates/parser/src/lexed_str.rs
+++ b/crates/parser/src/lexed_str.rs

@@ -122,8 +122,8 @@ impl<'a> LexedStr<'a> {

self.error.iter().map(|it| (it.token as usize, it.msg.as_str()))

}

- pub fn to_tokens(&self) -> crate::Tokens {

- let mut res = crate::Tokens::default();

+ pub fn to_input(&self) -> crate::Input {

+ let mut res = crate::Input::default();

let mut was_joint = false;

for i in 0..self.len() {

let kind = self.kind(i);

diff --git a/crates/parser/src/lib.rs b/crates/parser/src/lib.rs
index dc02ae6e83..da78889f35 100644
--- a/crates/parser/src/lib.rs
+++ b/crates/parser/src/lib.rs

@@ -24,32 +24,20 @@ mod syntax_kind;

mod event;

mod parser;

mod grammar;

-mod tokens;

+mod input;

+mod output;

#[cfg(test)]

mod tests;

pub(crate) use token_set::TokenSet;

-pub use crate::{lexed_str::LexedStr, syntax_kind::SyntaxKind, tokens::Tokens};

-#[derive(Debug, Clone, PartialEq, Eq, Hash)]

-pub struct ParseError(pub Box<String>);

-/// `TreeSink` abstracts details of a particular syntax tree implementation.

-pub trait TreeSink {

- /// Adds new token to the current branch.

- fn token(&mut self, kind: SyntaxKind, n_tokens: u8);

- /// Start new branch and make it current.

- fn start_node(&mut self, kind: SyntaxKind);

- /// Finish current branch and restore previous

- /// branch as current.

- fn finish_node(&mut self);

- fn error(&mut self, error: ParseError);

+pub use crate::{

+ input::Input,

+ lexed_str::LexedStr,

+ output::{Output, Step},

+ syntax_kind::SyntaxKind,

+};

/// rust-analyzer parser allows you to choose one of the possible entry points.

///

@@ -74,11 +62,19 @@ pub enum ParserEntryPoint {

}

/// Parse given tokens into the given sink as a rust file.

-pub fn parse_source_file(tokens: &Tokens, tree_sink: &mut dyn TreeSink) {

- parse(tokens, tree_sink, ParserEntryPoint::SourceFile);

+pub fn parse_source_file(inp: &Input) -> Output {

+ parse(inp, ParserEntryPoint::SourceFile)

}

-pub fn parse(tokens: &Tokens, tree_sink: &mut dyn TreeSink, entry_point: ParserEntryPoint) {

+/// Parses the given [`Input`] into [`Output`] assuming that the top-level

+/// syntactic construct is the given [`ParserEntryPoint`].

+///

+/// Both input and output here are fairly abstract. The overall flow is that the

+/// caller has some "real" tokens, converts them to [`Input`], parses them to

+/// [`Output`], and then converts that into a "real" tree. The "real" tree is

+/// made of "real" tokens, so this all hinges on rather tight coordination of

+/// indices between the four stages.

+pub fn parse(inp: &Input, entry_point: ParserEntryPoint) -> Output {

let entry_point: fn(&'_ mut parser::Parser) = match entry_point {

ParserEntryPoint::SourceFile => grammar::entry_points::source_file,

ParserEntryPoint::Path => grammar::entry_points::path,

@@ -96,10 +92,10 @@ pub fn parse(tokens: &Tokens, tree_sink: &mut dyn TreeSink, entry_point: ParserE

ParserEntryPoint::Attr => grammar::entry_points::attr,

};

- let mut p = parser::Parser::new(tokens);

+ let mut p = parser::Parser::new(inp);

entry_point(&mut p);

let events = p.finish();

- event::process(tree_sink, events);

+ event::process(events)

}

/// A parsing function for a specific braced-block.

@@ -119,11 +115,11 @@ impl Reparser {

///

/// Tokens must start with `{`, end with `}` and form a valid brace

/// sequence.

- pub fn parse(self, tokens: &Tokens, tree_sink: &mut dyn TreeSink) {

+ pub fn parse(self, tokens: &Input) -> Output {

let Reparser(r) = self;

let mut p = parser::Parser::new(tokens);

r(&mut p);

let events = p.finish();

- event::process(tree_sink, events);

+ event::process(events)

}

diff --git a/crates/parser/src/output.rs b/crates/parser/src/output.rs
new file mode 100644
index 0000000000..b613df029f
--- /dev/null
+++ b/crates/parser/src/output.rs

@@ -0,0 +1,76 @@

+//! See [`Output`]

+use crate::SyntaxKind;

+/// Output of the parser -- a DFS traversal of a concrete syntax tree.

+///

+/// Use the [`Output::iter`] method to iterate over traversal steps and consume

+/// a syntax tree.

+///

+/// In a sense, this is just a sequence of [`SyntaxKind`]-colored parenthesis

+/// interspersed into the original [`crate::Input`]. The output is fundamentally

+/// coordinated with the input and `n_input_tokens` refers to the number of

+/// times [`crate::Input::push`] was called.

+#[derive(Default)]

+pub struct Output {

+ /// 32-bit encoding of events. If LSB is zero, then that's an index into the

+ /// error vector. Otherwise, it's one of the thee other variants, with data encoded as

+ ///

+ ///

+ event: Vec<u32>,

+ error: Vec<String>,

+pub enum Step<'a> {

+ Token { kind: SyntaxKind, n_input_tokens: u8 },

+ Enter { kind: SyntaxKind },

+ Exit,

+ Error { msg: &'a str },

+impl Output {

+ pub fn iter(&self) -> impl Iterator<Item = Step<'_>> {

+ self.event.iter().map(|&event| {

+ if event & 0b1 == 0 {

+ return Step::Error { msg: self.error[(event as usize) >> 1].as_str() };

+ }

+ let tag = ((event & 0x0000_00F0) >> 4) as u8;

+ match tag {

+ 0 => {

+ let kind: SyntaxKind = (((event & 0xFFFF_0000) >> 16) as u16).into();

+ let n_input_tokens = ((event & 0x0000_FF00) >> 8) as u8;

+ Step::Token { kind, n_input_tokens }

+ }

+ 1 => {

+ let kind: SyntaxKind = (((event & 0xFFFF_0000) >> 16) as u16).into();

+ Step::Enter { kind }

+ }

+ 2 => Step::Exit,

+ _ => unreachable!(),

+ }

+ })

+ }

+ pub(crate) fn token(&mut self, kind: SyntaxKind, n_tokens: u8) {

+ let e = ((kind as u16 as u32) << 16) | ((n_tokens as u32) << 8) | (0 << 4) | 1;

+ self.event.push(e)

+ }

+ pub(crate) fn enter_node(&mut self, kind: SyntaxKind) {

+ let e = ((kind as u16 as u32) << 16) | (1 << 4) | 1;

+ self.event.push(e)

+ }

+ pub(crate) fn leave_node(&mut self) {

+ let e = 2 << 4 | 1;

+ self.event.push(e)

+ }

+ pub(crate) fn error(&mut self, error: String) {

+ let idx = self.error.len();

+ self.error.push(error);

+ let e = (idx as u32) << 1;

+ self.event.push(e);

+ }

diff --git a/crates/parser/src/parser.rs b/crates/parser/src/parser.rs
index 4c891108a6..d4aecf9b44 100644
--- a/crates/parser/src/parser.rs
+++ b/crates/parser/src/parser.rs

@@ -7,8 +7,7 @@ use limit::Limit;

use crate::{

event::Event,

- tokens::Tokens,

- ParseError,

+ input::Input,

SyntaxKind::{self, EOF, ERROR, TOMBSTONE},

TokenSet, T,

};

@@ -23,7 +22,7 @@ use crate::{

/// "start expression, consume number literal,

/// finish expression". See `Event` docs for more.

pub(crate) struct Parser<'t> {

- tokens: &'t Tokens,

+ inp: &'t Input,

pos: usize,

events: Vec<Event>,

steps: Cell<u32>,

@@ -32,8 +31,8 @@ pub(crate) struct Parser<'t> {

static PARSER_STEP_LIMIT: Limit = Limit::new(15_000_000);

impl<'t> Parser<'t> {

- pub(super) fn new(tokens: &'t Tokens) -> Parser<'t> {

- Parser { tokens, pos: 0, events: Vec::new(), steps: Cell::new(0) }

+ pub(super) fn new(inp: &'t Input) -> Parser<'t> {

+ Parser { inp, pos: 0, events: Vec::new(), steps: Cell::new(0) }

}

pub(crate) fn finish(self) -> Vec<Event> {

@@ -56,7 +55,7 @@ impl<'t> Parser<'t> {

assert!(PARSER_STEP_LIMIT.check(steps as usize).is_ok(), "the parser seems stuck");

self.steps.set(steps + 1);

- self.tokens.kind(self.pos + n)

+ self.inp.kind(self.pos + n)

}

/// Checks if the current token is `kind`.

@@ -92,7 +91,7 @@ impl<'t> Parser<'t> {

T![<<=] => self.at_composite3(n, T![<], T![<], T![=]),

T![>>=] => self.at_composite3(n, T![>], T![>], T![=]),

- _ => self.tokens.kind(self.pos + n) == kind,

+ _ => self.inp.kind(self.pos + n) == kind,

}

@@ -131,17 +130,17 @@ impl<'t> Parser<'t> {

}

fn at_composite2(&self, n: usize, k1: SyntaxKind, k2: SyntaxKind) -> bool {

- self.tokens.kind(self.pos + n) == k1

- && self.tokens.kind(self.pos + n + 1) == k2

- && self.tokens.is_joint(self.pos + n)

+ self.inp.kind(self.pos + n) == k1

+ && self.inp.kind(self.pos + n + 1) == k2

+ && self.inp.is_joint(self.pos + n)

}

fn at_composite3(&self, n: usize, k1: SyntaxKind, k2: SyntaxKind, k3: SyntaxKind) -> bool {

- self.tokens.kind(self.pos + n) == k1

- && self.tokens.kind(self.pos + n + 1) == k2

- && self.tokens.kind(self.pos + n + 2) == k3

- && self.tokens.is_joint(self.pos + n)

- && self.tokens.is_joint(self.pos + n + 1)

+ self.inp.kind(self.pos + n) == k1

+ && self.inp.kind(self.pos + n + 1) == k2

+ && self.inp.kind(self.pos + n + 2) == k3

+ && self.inp.is_joint(self.pos + n)

+ && self.inp.is_joint(self.pos + n + 1)

}

/// Checks if the current token is in `kinds`.

@@ -151,7 +150,7 @@ impl<'t> Parser<'t> {

/// Checks if the current token is contextual keyword with text `t`.

pub(crate) fn at_contextual_kw(&self, kw: SyntaxKind) -> bool {

- self.tokens.contextual_kind(self.pos) == kw

+ self.inp.contextual_kind(self.pos) == kw

}

/// Starts a new node in the syntax tree. All nodes and tokens

@@ -196,7 +195,7 @@ impl<'t> Parser<'t> {

/// structured errors with spans and notes, like rustc

/// does.

pub(crate) fn error<T: Into<String>>(&mut self, message: T) {

- let msg = ParseError(Box::new(message.into()));

+ let msg = message.into();

self.push_event(Event::Error { msg });

}

diff --git a/crates/syntax/src/parsing.rs b/crates/syntax/src/parsing.rs
index cba1ddde85..20c7101a03 100644
--- a/crates/syntax/src/parsing.rs
+++ b/crates/syntax/src/parsing.rs

@@ -4,24 +4,18 @@

mod text_tree_sink;

mod reparsing;

-use parser::SyntaxKind;

-use text_tree_sink::TextTreeSink;

-use crate::{syntax_node::GreenNode, AstNode, SyntaxError, SyntaxNode};

+use crate::{

+ parsing::text_tree_sink::build_tree, syntax_node::GreenNode, AstNode, SyntaxError, SyntaxNode,

+};

pub(crate) use crate::parsing::reparsing::incremental_reparse;

pub(crate) fn parse_text(text: &str) -> (GreenNode, Vec<SyntaxError>) {

let lexed = parser::LexedStr::new(text);

- let parser_tokens = lexed.to_tokens();

- let mut tree_sink = TextTreeSink::new(lexed);

- parser::parse_source_file(&parser_tokens, &mut tree_sink);

- let (tree, parser_errors) = tree_sink.finish();

- (tree, parser_errors)

+ let parser_input = lexed.to_input();

+ let parser_output = parser::parse_source_file(&parser_input);

+ let (node, errors, _eof) = build_tree(lexed, parser_output, false);

+ (node, errors)

}

/// Returns `text` parsed as a `T` provided there are no parse errors.

@@ -33,21 +27,13 @@ pub(crate) fn parse_text_as<T: AstNode>(

if lexed.errors().next().is_some() {

return Err(());

}

- let parser_tokens = lexed.to_tokens();

- let mut tree_sink = TextTreeSink::new(lexed);

- // TextTreeSink assumes that there's at least some root node to which it can attach errors and

- // tokens. We arbitrarily give it a SourceFile.

- use parser::TreeSink;

- tree_sink.start_node(SyntaxKind::SOURCE_FILE);

- parser::parse(&parser_tokens, &mut tree_sink, entry_point);

- tree_sink.finish_node();

+ let parser_input = lexed.to_input();

+ let parser_output = parser::parse(&parser_input, entry_point);

+ let (node, errors, eof) = build_tree(lexed, parser_output, true);

- let (tree, parser_errors, eof) = tree_sink.finish_eof();

- if !parser_errors.is_empty() || !eof {

+ if !errors.is_empty() || !eof {

return Err(());

}

- SyntaxNode::new_root(tree).first_child().and_then(T::cast).ok_or(())

+ SyntaxNode::new_root(node).first_child().and_then(T::cast).ok_or(())

}

diff --git a/crates/syntax/src/parsing/reparsing.rs b/crates/syntax/src/parsing/reparsing.rs
index e9567a838c..a6abe3cccf 100644
--- a/crates/syntax/src/parsing/reparsing.rs
+++ b/crates/syntax/src/parsing/reparsing.rs

@@ -10,7 +10,7 @@ use parser::Reparser;

use text_edit::Indel;

use crate::{

- parsing::text_tree_sink::TextTreeSink,

+ parsing::text_tree_sink::build_tree,

syntax_node::{GreenNode, GreenToken, NodeOrToken, SyntaxElement, SyntaxNode},

SyntaxError,

SyntaxKind::*,

@@ -89,16 +89,14 @@ fn reparse_block(

let text = get_text_after_edit(node.clone().into(), edit);

let lexed = parser::LexedStr::new(text.as_str());

- let parser_tokens = lexed.to_tokens();

+ let parser_input = lexed.to_input();

if !is_balanced(&lexed) {

return None;

}

- let mut tree_sink = TextTreeSink::new(lexed);

+ let tree_traversal = reparser.parse(&parser_input);

- reparser.parse(&parser_tokens, &mut tree_sink);

- let (green, new_parser_errors) = tree_sink.finish();

+ let (green, new_parser_errors, _eof) = build_tree(lexed, tree_traversal, false);

Some((node.replace_with(green), new_parser_errors, node.text_range()))

}

diff --git a/crates/syntax/src/parsing/text_tree_sink.rs b/crates/syntax/src/parsing/text_tree_sink.rs
index c9e7feb965..f40c549e3d 100644
--- a/crates/syntax/src/parsing/text_tree_sink.rs
+++ b/crates/syntax/src/parsing/text_tree_sink.rs

@@ -2,7 +2,7 @@

use std::mem;

-use parser::{LexedStr, ParseError, TreeSink};

+use parser::LexedStr;

use crate::{

ast,

@@ -12,10 +12,37 @@ use crate::{

SyntaxTreeBuilder, TextRange,

};

-/// Bridges the parser with our specific syntax tree representation.

-///

-/// `TextTreeSink` also handles attachment of trivia (whitespace) to nodes.

-pub(crate) struct TextTreeSink<'a> {

+pub(crate) fn build_tree(

+ lexed: LexedStr<'_>,

+ parser_output: parser::Output,

+ synthetic_root: bool,

+) -> (GreenNode, Vec<SyntaxError>, bool) {

+ let mut builder = Builder::new(lexed);

+ if synthetic_root {

+ builder.enter(SyntaxKind::SOURCE_FILE);

+ }

+ for event in parser_output.iter() {

+ match event {

+ parser::Step::Token { kind, n_input_tokens: n_raw_tokens } => {

+ builder.token(kind, n_raw_tokens)

+ }

+ parser::Step::Enter { kind } => builder.enter(kind),

+ parser::Step::Exit => builder.exit(),

+ parser::Step::Error { msg } => {

+ let text_pos = builder.lexed.text_start(builder.pos).try_into().unwrap();

+ builder.inner.error(msg.to_string(), text_pos);

+ }

+ if synthetic_root {

+ builder.exit()

+ }

+ builder.build()

+struct Builder<'a> {

lexed: LexedStr<'a>,

pos: usize,

state: State,

@@ -28,7 +55,35 @@ enum State {

PendingFinish,

}

-impl<'a> TreeSink for TextTreeSink<'a> {

+impl<'a> Builder<'a> {

+ fn new(lexed: parser::LexedStr<'a>) -> Self {

+ Self { lexed, pos: 0, state: State::PendingStart, inner: SyntaxTreeBuilder::default() }

+ }

+ fn build(mut self) -> (GreenNode, Vec<SyntaxError>, bool) {

+ match mem::replace(&mut self.state, State::Normal) {

+ State::PendingFinish => {

+ self.eat_trivias();

+ self.inner.finish_node();

+ }

+ State::PendingStart | State::Normal => unreachable!(),

+ }

+ let (node, mut errors) = self.inner.finish_raw();

+ for (i, err) in self.lexed.errors() {

+ let text_range = self.lexed.text_range(i);

+ let text_range = TextRange::new(

+ text_range.start.try_into().unwrap(),

+ text_range.end.try_into().unwrap(),

+ );

+ errors.push(SyntaxError::new(err, text_range))

+ }

+ let is_eof = self.pos == self.lexed.len();

+ (node, errors, is_eof)

+ }

fn token(&mut self, kind: SyntaxKind, n_tokens: u8) {

match mem::replace(&mut self.state, State::Normal) {

State::PendingStart => unreachable!(),

@@ -39,7 +94,7 @@ impl<'a> TreeSink for TextTreeSink<'a> {

self.do_token(kind, n_tokens as usize);

}

- fn start_node(&mut self, kind: SyntaxKind) {

+ fn enter(&mut self, kind: SyntaxKind) {

match mem::replace(&mut self.state, State::Normal) {

State::PendingStart => {

self.inner.start_node(kind);

@@ -63,7 +118,7 @@ impl<'a> TreeSink for TextTreeSink<'a> {

self.eat_n_trivias(n_attached_trivias);

}

- fn finish_node(&mut self) {

+ fn exit(&mut self) {

match mem::replace(&mut self.state, State::PendingFinish) {

State::PendingStart => unreachable!(),

State::PendingFinish => self.inner.finish_node(),

@@ -71,46 +126,6 @@ impl<'a> TreeSink for TextTreeSink<'a> {

}

- fn error(&mut self, error: ParseError) {

- let text_pos = self.lexed.text_start(self.pos).try_into().unwrap();

- self.inner.error(error, text_pos);

- }

-impl<'a> TextTreeSink<'a> {

- pub(super) fn new(lexed: parser::LexedStr<'a>) -> Self {

- Self { lexed, pos: 0, state: State::PendingStart, inner: SyntaxTreeBuilder::default() }

- }

- pub(super) fn finish_eof(mut self) -> (GreenNode, Vec<SyntaxError>, bool) {

- match mem::replace(&mut self.state, State::Normal) {

- State::PendingFinish => {

- self.eat_trivias();

- self.inner.finish_node();

- }

- State::PendingStart | State::Normal => unreachable!(),

- }

- let (node, mut errors) = self.inner.finish_raw();

- for (i, err) in self.lexed.errors() {

- let text_range = self.lexed.text_range(i);

- let text_range = TextRange::new(

- text_range.start.try_into().unwrap(),

- text_range.end.try_into().unwrap(),

- );

- errors.push(SyntaxError::new(err, text_range))

- }

- let is_eof = self.pos == self.lexed.len();

- (node, errors, is_eof)

- }

- pub(super) fn finish(self) -> (GreenNode, Vec<SyntaxError>) {

- let (node, errors, _eof) = self.finish_eof();

- (node, errors)

- }

fn eat_trivias(&mut self) {

while self.pos < self.lexed.len() {

let kind = self.lexed.kind(self.pos);

diff --git a/crates/syntax/src/syntax_node.rs b/crates/syntax/src/syntax_node.rs
index c95c76c0a8..b96f10c173 100644
--- a/crates/syntax/src/syntax_node.rs
+++ b/crates/syntax/src/syntax_node.rs

@@ -69,7 +69,7 @@ impl SyntaxTreeBuilder {

self.inner.finish_node();

}

- pub fn error(&mut self, error: parser::ParseError, text_pos: TextSize) {

- self.errors.push(SyntaxError::new_at_offset(*error.0, text_pos));

+ pub fn error(&mut self, error: String, text_pos: TextSize) {

+ self.errors.push(SyntaxError::new_at_offset(error, text_pos));

}