Unnamed repository; edit this file 'description' to name the repository.
simplify
| -rw-r--r-- | crates/parser/src/parser.rs | 30 | ||||
| -rw-r--r-- | crates/parser/src/tokens.rs | 40 |
2 files changed, 23 insertions, 47 deletions
diff --git a/crates/parser/src/parser.rs b/crates/parser/src/parser.rs index 759f87f496..4c891108a6 100644 --- a/crates/parser/src/parser.rs +++ b/crates/parser/src/parser.rs @@ -56,7 +56,7 @@ impl<'t> Parser<'t> { assert!(PARSER_STEP_LIMIT.check(steps as usize).is_ok(), "the parser seems stuck"); self.steps.set(steps + 1); - self.tokens.get(self.pos + n).kind + self.tokens.kind(self.pos + n) } /// Checks if the current token is `kind`. @@ -92,7 +92,7 @@ impl<'t> Parser<'t> { T![<<=] => self.at_composite3(n, T![<], T![<], T![=]), T![>>=] => self.at_composite3(n, T![>], T![>], T![=]), - _ => self.tokens.get(self.pos + n).kind == kind, + _ => self.tokens.kind(self.pos + n) == kind, } } @@ -131,25 +131,17 @@ impl<'t> Parser<'t> { } fn at_composite2(&self, n: usize, k1: SyntaxKind, k2: SyntaxKind) -> bool { - let t1 = self.tokens.get(self.pos + n); - if t1.kind != k1 || !t1.is_jointed_to_next { - return false; - } - let t2 = self.tokens.get(self.pos + n + 1); - t2.kind == k2 + self.tokens.kind(self.pos + n) == k1 + && self.tokens.kind(self.pos + n + 1) == k2 + && self.tokens.is_joint(self.pos + n) } fn at_composite3(&self, n: usize, k1: SyntaxKind, k2: SyntaxKind, k3: SyntaxKind) -> bool { - let t1 = self.tokens.get(self.pos + n); - if t1.kind != k1 || !t1.is_jointed_to_next { - return false; - } - let t2 = self.tokens.get(self.pos + n + 1); - if t2.kind != k2 || !t2.is_jointed_to_next { - return false; - } - let t3 = self.tokens.get(self.pos + n + 2); - t3.kind == k3 + self.tokens.kind(self.pos + n) == k1 + && self.tokens.kind(self.pos + n + 1) == k2 + && self.tokens.kind(self.pos + n + 2) == k3 + && self.tokens.is_joint(self.pos + n) + && self.tokens.is_joint(self.pos + n + 1) } /// Checks if the current token is in `kinds`. @@ -159,7 +151,7 @@ impl<'t> Parser<'t> { /// Checks if the current token is contextual keyword with text `t`. pub(crate) fn at_contextual_kw(&self, kw: SyntaxKind) -> bool { - self.tokens.get(self.pos).contextual_kw == kw + self.tokens.contextual_kind(self.pos) == kw } /// Starts a new node in the syntax tree. All nodes and tokens diff --git a/crates/parser/src/tokens.rs b/crates/parser/src/tokens.rs index 1128cfe99d..74725df6d0 100644 --- a/crates/parser/src/tokens.rs +++ b/crates/parser/src/tokens.rs @@ -8,17 +8,6 @@ use crate::SyntaxKind; #[allow(non_camel_case_types)] type bits = u64; -/// `Token` abstracts the cursor of `TokenSource` operates on. -#[derive(Debug, Copy, Clone, Eq, PartialEq)] -pub(crate) struct Token { - /// What is the current token? - pub(crate) kind: SyntaxKind, - - /// Is the current token joined to the next one (`> >` vs `>>`). - pub(crate) is_jointed_to_next: bool, - pub(crate) contextual_kw: SyntaxKind, -} - /// Main input to the parser. /// /// A sequence of tokens represented internally as a struct of arrays. @@ -71,10 +60,6 @@ impl Tokens { let (idx, b_idx) = self.bit_index(n); self.joint[idx] |= 1 << b_idx; } - fn get_joint(&self, n: usize) -> bool { - let (idx, b_idx) = self.bit_index(n); - self.joint[idx] & 1 << b_idx != 0 - } fn bit_index(&self, n: usize) -> (usize, usize) { let idx = n / (bits::BITS as usize); let b_idx = n % (bits::BITS as usize); @@ -84,19 +69,18 @@ impl Tokens { fn len(&self) -> usize { self.kind.len() } - pub(crate) fn get(&self, idx: usize) -> Token { - if idx < self.len() { - let kind = self.kind[idx]; - let is_jointed_to_next = self.get_joint(idx); - let contextual_kw = self.contextual_kw[idx]; - Token { kind, is_jointed_to_next, contextual_kw } - } else { - self.eof() - } - } +} - #[cold] - fn eof(&self) -> Token { - Token { kind: SyntaxKind::EOF, is_jointed_to_next: false, contextual_kw: SyntaxKind::EOF } +/// pub(crate) impl used by the parser. +impl Tokens { + pub(crate) fn kind(&self, idx: usize) -> SyntaxKind { + self.kind.get(idx).copied().unwrap_or(SyntaxKind::EOF) + } + pub(crate) fn contextual_kind(&self, idx: usize) -> SyntaxKind { + self.contextual_kw.get(idx).copied().unwrap_or(SyntaxKind::EOF) + } + pub(crate) fn is_joint(&self, n: usize) -> bool { + let (idx, b_idx) = self.bit_index(n); + self.joint[idx] & 1 << b_idx != 0 } } |