Unnamed repository; edit this file 'description' to name the repository.
Diffstat (limited to 'crates/parser/src/tokens.rs')
| -rw-r--r-- | crates/parser/src/tokens.rs | 26 |
1 files changed, 19 insertions, 7 deletions
diff --git a/crates/parser/src/tokens.rs b/crates/parser/src/tokens.rs index 495d9713ea..4f10956070 100644 --- a/crates/parser/src/tokens.rs +++ b/crates/parser/src/tokens.rs @@ -1,8 +1,19 @@ -use crate::{SyntaxKind, Token}; +use crate::SyntaxKind; #[allow(non_camel_case_types)] type bits = u64; +/// `Token` abstracts the cursor of `TokenSource` operates on. +#[derive(Debug, Copy, Clone, Eq, PartialEq)] +pub(crate) struct Token { + /// What is the current token? + pub(crate) kind: SyntaxKind, + + /// Is the current token joined to the next one (`> >` vs `>>`). + pub(crate) is_jointed_to_next: bool, + pub(crate) contextual_kw: SyntaxKind, +} + /// Main input to the parser. /// /// A sequence of tokens represented internally as a struct of arrays. @@ -49,13 +60,14 @@ impl Tokens { self.kind.len() } pub(crate) fn get(&self, idx: usize) -> Token { - if idx > self.len() { - return self.eof(); + if idx < self.len() { + let kind = self.kind[idx]; + let is_jointed_to_next = self.get_joint(idx); + let contextual_kw = self.contextual_kw[idx]; + Token { kind, is_jointed_to_next, contextual_kw } + } else { + self.eof() } - let kind = self.kind[idx]; - let is_jointed_to_next = self.get_joint(idx); - let contextual_kw = self.contextual_kw[idx]; - Token { kind, is_jointed_to_next, contextual_kw } } #[cold] |