bendns' repos / rust-analyzer
Unnamed repository; edit this file 'description' to name the repository.
Diffstat (limited to 'crates/parser/src/tokens.rs')
-rw-r--r--crates/parser/src/tokens.rs40
1 files changed, 12 insertions, 28 deletions
diff --git a/crates/parser/src/tokens.rs b/crates/parser/src/tokens.rs
index 1128cfe99d..74725df6d0 100644
--- a/crates/parser/src/tokens.rs
+++ b/crates/parser/src/tokens.rs
@@ -8,17 +8,6 @@ use crate::SyntaxKind;
#[allow(non_camel_case_types)]
type bits = u64;
-/// `Token` abstracts the cursor of `TokenSource` operates on.
-#[derive(Debug, Copy, Clone, Eq, PartialEq)]
-pub(crate) struct Token {
- /// What is the current token?
- pub(crate) kind: SyntaxKind,
-
- /// Is the current token joined to the next one (`> >` vs `>>`).
- pub(crate) is_jointed_to_next: bool,
- pub(crate) contextual_kw: SyntaxKind,
-}
-
/// Main input to the parser.
///
/// A sequence of tokens represented internally as a struct of arrays.
@@ -71,10 +60,6 @@ impl Tokens {
let (idx, b_idx) = self.bit_index(n);
self.joint[idx] |= 1 << b_idx;
}
- fn get_joint(&self, n: usize) -> bool {
- let (idx, b_idx) = self.bit_index(n);
- self.joint[idx] & 1 << b_idx != 0
- }
fn bit_index(&self, n: usize) -> (usize, usize) {
let idx = n / (bits::BITS as usize);
let b_idx = n % (bits::BITS as usize);
@@ -84,19 +69,18 @@ impl Tokens {
fn len(&self) -> usize {
self.kind.len()
}
- pub(crate) fn get(&self, idx: usize) -> Token {
- if idx < self.len() {
- let kind = self.kind[idx];
- let is_jointed_to_next = self.get_joint(idx);
- let contextual_kw = self.contextual_kw[idx];
- Token { kind, is_jointed_to_next, contextual_kw }
- } else {
- self.eof()
- }
- }
+}
- #[cold]
- fn eof(&self) -> Token {
- Token { kind: SyntaxKind::EOF, is_jointed_to_next: false, contextual_kw: SyntaxKind::EOF }
+/// pub(crate) impl used by the parser.
+impl Tokens {
+ pub(crate) fn kind(&self, idx: usize) -> SyntaxKind {
+ self.kind.get(idx).copied().unwrap_or(SyntaxKind::EOF)
+ }
+ pub(crate) fn contextual_kind(&self, idx: usize) -> SyntaxKind {
+ self.contextual_kw.get(idx).copied().unwrap_or(SyntaxKind::EOF)
+ }
+ pub(crate) fn is_joint(&self, n: usize) -> bool {
+ let (idx, b_idx) = self.bit_index(n);
+ self.joint[idx] & 1 << b_idx != 0
}
}