Unnamed repository; edit this file 'description' to name the repository.
tighten up invariants
Aleksey Kladov 2021-12-12
parent 18d4737 · commit 57e6ef0
-rw-r--r--crates/mbe/src/to_parser_tokens.rs4
-rw-r--r--crates/parser/src/tokens.rs7
-rw-r--r--crates/syntax/src/parsing.rs20
3 files changed, 17 insertions, 14 deletions
diff --git a/crates/mbe/src/to_parser_tokens.rs b/crates/mbe/src/to_parser_tokens.rs
index 435226342e..644689f432 100644
--- a/crates/mbe/src/to_parser_tokens.rs
+++ b/crates/mbe/src/to_parser_tokens.rs
@@ -62,7 +62,9 @@ pub(crate) fn to_parser_tokens(buffer: &TokenBuffer) -> parser::Tokens {
let kind = SyntaxKind::from_char(punct.char)
.unwrap_or_else(|| panic!("{:#?} is not a valid punct", punct));
res.push(kind);
- res.was_joint(punct.spacing == tt::Spacing::Joint);
+ if punct.spacing == tt::Spacing::Joint {
+ res.was_joint();
+ }
}
}
cursor.bump()
diff --git a/crates/parser/src/tokens.rs b/crates/parser/src/tokens.rs
index de831f0f70..1128cfe99d 100644
--- a/crates/parser/src/tokens.rs
+++ b/crates/parser/src/tokens.rs
@@ -51,11 +51,8 @@ impl Tokens {
/// tokens.push(curr_joint)
/// ```
#[inline]
- pub fn was_joint(&mut self, yes: bool) {
- let idx = self.len();
- if yes && idx > 0 {
- self.set_joint(idx - 1);
- }
+ pub fn was_joint(&mut self) {
+ self.set_joint(self.len() - 1);
}
#[inline]
pub fn push_ident(&mut self, contextual_kw: SyntaxKind) {
diff --git a/crates/syntax/src/parsing.rs b/crates/syntax/src/parsing.rs
index 5cafe70dd7..865e146482 100644
--- a/crates/syntax/src/parsing.rs
+++ b/crates/syntax/src/parsing.rs
@@ -58,18 +58,22 @@ pub(crate) fn parse_text_as<T: AstNode>(
pub(crate) fn to_parser_tokens(text: &str, lexer_tokens: &[lexer::Token]) -> ::parser::Tokens {
let mut off = 0;
let mut res = parser::Tokens::default();
- let mut was_joint = true;
+ let mut was_joint = false;
for t in lexer_tokens {
if t.kind.is_trivia() {
was_joint = false;
- } else if t.kind == SyntaxKind::IDENT {
- let token_text = &text[off..][..usize::from(t.len)];
- let contextual_kw =
- SyntaxKind::from_contextual_keyword(token_text).unwrap_or(SyntaxKind::IDENT);
- res.push_ident(contextual_kw);
} else {
- res.was_joint(was_joint);
- res.push(t.kind);
+ if t.kind == SyntaxKind::IDENT {
+ let token_text = &text[off..][..usize::from(t.len)];
+ let contextual_kw =
+ SyntaxKind::from_contextual_keyword(token_text).unwrap_or(SyntaxKind::IDENT);
+ res.push_ident(contextual_kw);
+ } else {
+ if was_joint {
+ res.was_joint();
+ }
+ res.push(t.kind);
+ }
was_joint = true;
}
off += usize::from(t.len);