bendns' repos / rust-analyzer
Unnamed repository; edit this file 'description' to name the repository.
Diffstat (limited to 'crates/parser/src/token_set.rs')
-rw-r--r--crates/parser/src/token_set.rs34
1 files changed, 24 insertions, 10 deletions
diff --git a/crates/parser/src/token_set.rs b/crates/parser/src/token_set.rs
index cd4894c1e8..88a89a53a7 100644
--- a/crates/parser/src/token_set.rs
+++ b/crates/parser/src/token_set.rs
@@ -4,34 +4,48 @@ use crate::SyntaxKind;
/// A bit-set of `SyntaxKind`s
#[derive(Clone, Copy)]
-pub(crate) struct TokenSet(u128);
+pub(crate) struct TokenSet([u64; 3]);
+
+/// `TokenSet`s should only include token `SyntaxKind`s, so the discriminant of any passed/included
+/// `SyntaxKind` must *not* be greater than that of the last token `SyntaxKind`.
+/// See #17037.
+const LAST_TOKEN_KIND_DISCRIMINANT: usize = SyntaxKind::SHEBANG as usize;
impl TokenSet {
- pub(crate) const EMPTY: TokenSet = TokenSet(0);
+ pub(crate) const EMPTY: TokenSet = TokenSet([0; 3]);
pub(crate) const fn new(kinds: &[SyntaxKind]) -> TokenSet {
- let mut res = 0u128;
+ let mut res = [0; 3];
let mut i = 0;
while i < kinds.len() {
- res |= mask(kinds[i]);
+ let discriminant = kinds[i] as usize;
+ debug_assert!(
+ discriminant <= LAST_TOKEN_KIND_DISCRIMINANT,
+ "Expected a token `SyntaxKind`"
+ );
+ let idx = discriminant / 64;
+ res[idx] |= 1 << (discriminant % 64);
i += 1;
}
TokenSet(res)
}
pub(crate) const fn union(self, other: TokenSet) -> TokenSet {
- TokenSet(self.0 | other.0)
+ TokenSet([self.0[0] | other.0[0], self.0[1] | other.0[1], self.0[2] | other.0[2]])
}
pub(crate) const fn contains(&self, kind: SyntaxKind) -> bool {
- self.0 & mask(kind) != 0
+ let discriminant = kind as usize;
+ debug_assert!(
+ discriminant <= LAST_TOKEN_KIND_DISCRIMINANT,
+ "Expected a token `SyntaxKind`"
+ );
+ let idx = discriminant / 64;
+ let mask = 1 << (discriminant % 64);
+ self.0[idx] & mask != 0
}
}
-const fn mask(kind: SyntaxKind) -> u128 {
- 1u128 << (kind as usize)
-}
-
#[test]
fn token_set_works_for_tokens() {
use crate::SyntaxKind::*;