Unnamed repository; edit this file 'description' to name the repository.
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
use crate::SyntaxKind;

#[allow(non_camel_case_types)]
type bits = u64;

/// `Token` abstracts the cursor of `TokenSource` operates on.
#[derive(Debug, Copy, Clone, Eq, PartialEq)]
pub(crate) struct Token {
    /// What is the current token?
    pub(crate) kind: SyntaxKind,

    /// Is the current token joined to the next one (`> >` vs `>>`).
    pub(crate) is_jointed_to_next: bool,
    pub(crate) contextual_kw: SyntaxKind,
}

/// Main input to the parser.
///
/// A sequence of tokens represented internally as a struct of arrays.
#[derive(Default)]
pub struct Tokens {
    kind: Vec<SyntaxKind>,
    joint: Vec<bits>,
    contextual_kw: Vec<SyntaxKind>,
}

impl Tokens {
    pub fn push(&mut self, kind: SyntaxKind) {
        self.push_impl(kind, SyntaxKind::EOF)
    }
    pub fn was_joint(&mut self, yes: bool) {
        let idx = self.len();
        if yes && idx > 0 {
            self.set_joint(idx - 1);
        }
    }
    pub fn push_ident(&mut self, contextual_kw: SyntaxKind) {
        self.push_impl(SyntaxKind::IDENT, contextual_kw)
    }
    fn push_impl(&mut self, kind: SyntaxKind, contextual_kw: SyntaxKind) {
        let idx = self.len();
        if idx % (bits::BITS as usize) == 0 {
            self.joint.push(0);
        }
        self.kind.push(kind);
        self.contextual_kw.push(contextual_kw);
    }
    fn set_joint(&mut self, n: usize) {
        let (idx, b_idx) = self.bit_index(n);
        self.joint[idx] |= 1 << b_idx;
    }
    fn get_joint(&self, n: usize) -> bool {
        let (idx, b_idx) = self.bit_index(n);
        self.joint[idx] & 1 << b_idx != 0
    }
    fn bit_index(&self, n: usize) -> (usize, usize) {
        let idx = n / (bits::BITS as usize);
        let b_idx = n % (bits::BITS as usize);
        (idx, b_idx)
    }

    pub fn len(&self) -> usize {
        self.kind.len()
    }
    pub(crate) fn get(&self, idx: usize) -> Token {
        if idx < self.len() {
            let kind = self.kind[idx];
            let is_jointed_to_next = self.get_joint(idx);
            let contextual_kw = self.contextual_kw[idx];
            Token { kind, is_jointed_to_next, contextual_kw }
        } else {
            self.eof()
        }
    }

    #[cold]
    fn eof(&self) -> Token {
        Token { kind: SyntaxKind::EOF, is_jointed_to_next: false, contextual_kw: SyntaxKind::EOF }
    }
}