Unnamed repository; edit this file 'description' to name the repository.
Merge #11046
11046: internal: move all the lexing to the parser crate r=matklad a=matklad
bors r+
π€
Co-authored-by: Aleksey Kladov <[email protected]>
233 files changed, 971 insertions, 1019 deletions
diff --git a/Cargo.lock b/Cargo.lock index dca35a212a..44ef223557 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -609,6 +609,7 @@ dependencies = [ "hir", "ide_db", "itertools", + "parser", "profile", "rustc-hash", "sourcegen", @@ -654,6 +655,7 @@ dependencies = [ "itertools", "limit", "once_cell", + "parser", "profile", "rayon", "rustc-hash", @@ -695,6 +697,7 @@ dependencies = [ "hir", "ide_db", "itertools", + "parser", "rustc-hash", "syntax", "test_utils", @@ -1066,7 +1069,9 @@ name = "parser" version = "0.0.0" dependencies = [ "drop_bomb", + "expect-test", "limit", + "rustc-ap-rustc_lexer", ] [[package]] diff --git a/crates/ide_assists/Cargo.toml b/crates/ide_assists/Cargo.toml index 4d97e9150e..3cd186fdf5 100644 --- a/crates/ide_assists/Cargo.toml +++ b/crates/ide_assists/Cargo.toml @@ -16,6 +16,7 @@ itertools = "0.10.0" either = "1.6.1" stdx = { path = "../stdx", version = "0.0.0" } +parser = { path = "../parser", version = "0.0.0" } syntax = { path = "../syntax", version = "0.0.0" } text_edit = { path = "../text_edit", version = "0.0.0" } profile = { path = "../profile", version = "0.0.0" } diff --git a/crates/ide_assists/src/utils/suggest_name.rs b/crates/ide_assists/src/utils/suggest_name.rs index 2021db3aba..f91b2fe44e 100644 --- a/crates/ide_assists/src/utils/suggest_name.rs +++ b/crates/ide_assists/src/utils/suggest_name.rs @@ -135,7 +135,7 @@ fn normalize(name: &str) -> Option<String> { } fn is_valid_name(name: &str) -> bool { - match syntax::lex_single_syntax_kind(name) { + match parser::LexedStr::single_token(name) { Some((syntax::SyntaxKind::IDENT, _error)) => true, _ => false, } diff --git a/crates/ide_db/Cargo.toml b/crates/ide_db/Cargo.toml index ea20f5372c..cfcf9f56c8 100644 --- a/crates/ide_db/Cargo.toml +++ b/crates/ide_db/Cargo.toml @@ -22,6 +22,7 @@ arrayvec = "0.7" indexmap = "1.7" stdx = { path = "../stdx", version = "0.0.0" } +parser = { path = "../parser", version = "0.0.0" } syntax = { path = "../syntax", version = "0.0.0" } text_edit = { path = "../text_edit", version = "0.0.0" } base_db = { path = "../base_db", version = "0.0.0" } diff --git a/crates/ide_db/src/rename.rs b/crates/ide_db/src/rename.rs index 60160f9553..188499db72 100644 --- a/crates/ide_db/src/rename.rs +++ b/crates/ide_db/src/rename.rs @@ -28,7 +28,7 @@ use hir::{AsAssocItem, FieldSource, HasSource, InFile, ModuleSource, Semantics}; use stdx::never; use syntax::{ ast::{self, HasName}, - lex_single_syntax_kind, AstNode, SyntaxKind, TextRange, T, + AstNode, SyntaxKind, TextRange, T, }; use text_edit::{TextEdit, TextEditBuilder}; @@ -490,7 +490,7 @@ pub enum IdentifierKind { impl IdentifierKind { pub fn classify(new_name: &str) -> Result<IdentifierKind> { - match lex_single_syntax_kind(new_name) { + match parser::LexedStr::single_token(new_name) { Some(res) => match res { (SyntaxKind::IDENT, _) => Ok(IdentifierKind::Ident), (T![_], _) => Ok(IdentifierKind::Underscore), diff --git a/crates/ide_ssr/Cargo.toml b/crates/ide_ssr/Cargo.toml index efa8fd243a..9a8221ac6c 100644 --- a/crates/ide_ssr/Cargo.toml +++ b/crates/ide_ssr/Cargo.toml @@ -16,6 +16,7 @@ rustc-hash = "1.1.0" itertools = "0.10.0" text_edit = { path = "../text_edit", version = "0.0.0" } +parser = { path = "../parser", version = "0.0.0" } syntax = { path = "../syntax", version = "0.0.0" } ide_db = { path = "../ide_db", version = "0.0.0" } hir = { path = "../hir", version = "0.0.0" } diff --git a/crates/ide_ssr/src/parsing.rs b/crates/ide_ssr/src/parsing.rs index ed7c033e27..ae7d5b4bf1 100644 --- a/crates/ide_ssr/src/parsing.rs +++ b/crates/ide_ssr/src/parsing.rs @@ -256,19 +256,13 @@ fn validate_rule(rule: &SsrRule) -> Result<(), SsrError> { } fn tokenize(source: &str) -> Result<Vec<Token>, SsrError> { - let mut start = 0; - let (raw_tokens, errors) = syntax::tokenize(source); - if let Some(first_error) = errors.first() { + let lexed = parser::LexedStr::new(source); + if let Some((_, first_error)) = lexed.errors().next() { bail!("Failed to parse pattern: {}", first_error); } let mut tokens: Vec<Token> = Vec::new(); - for raw_token in raw_tokens { - let token_len = usize::from(raw_token.len); - tokens.push(Token { - kind: raw_token.kind, - text: SmolStr::new(&source[start..start + token_len]), - }); - start += token_len; + for i in 0..lexed.len() { + tokens.push(Token { kind: lexed.kind(i), text: lexed.text(i).into() }); } Ok(tokens) } diff --git a/crates/mbe/src/syntax_bridge.rs b/crates/mbe/src/syntax_bridge.rs index 28a23f6be2..109842b0cd 100644 --- a/crates/mbe/src/syntax_bridge.rs +++ b/crates/mbe/src/syntax_bridge.rs @@ -4,10 +4,9 @@ use parser::{ParseError, TreeSink}; use rustc_hash::{FxHashMap, FxHashSet}; use syntax::{ ast::{self, make::tokens::doc_comment}, - tokenize, AstToken, Parse, PreorderWithTokens, SmolStr, SyntaxElement, SyntaxKind, + AstToken, Parse, PreorderWithTokens, SmolStr, SyntaxElement, SyntaxKind, SyntaxKind::*, - SyntaxNode, SyntaxToken, SyntaxTreeBuilder, TextRange, TextSize, Token as RawToken, WalkEvent, - T, + SyntaxNode, SyntaxToken, SyntaxTreeBuilder, TextRange, TextSize, WalkEvent, T, }; use tt::buffer::{Cursor, TokenBuffer}; @@ -69,15 +68,14 @@ pub fn token_tree_to_syntax_node( /// Convert a string to a `TokenTree` pub fn parse_to_token_tree(text: &str) -> Option<(tt::Subtree, TokenMap)> { - let (tokens, errors) = tokenize(text); - if !errors.is_empty() { + let lexed = parser::LexedStr::new(text); + if lexed.errors().next().is_some() { return None; } let mut conv = RawConvertor { - text, - offset: TextSize::default(), - inner: tokens.iter(), + lexed: lexed, + pos: 0, id_alloc: TokenIdAlloc { map: Default::default(), global_offset: TextSize::default(), @@ -146,7 +144,7 @@ fn convert_tokens<C: TokenConvertor>(conv: &mut C) -> tt::Subtree { Some(it) => it, }; - let k: SyntaxKind = token.kind(); + let k: SyntaxKind = token.kind(&conv); if k == COMMENT { if let Some(tokens) = conv.convert_doc_comment(&token) { // FIXME: There has to be a better way to do this @@ -199,19 +197,19 @@ fn convert_tokens<C: TokenConvertor>(conv: &mut C) -> tt::Subtree { } else { let spacing = match conv.peek() { Some(next) - if next.kind().is_trivia() - || next.kind() == T!['['] - || next.kind() == T!['{'] - || next.kind() == T!['('] => + if next.kind(&conv).is_trivia() + || next.kind(&conv) == T!['['] + || next.kind(&conv) == T!['{'] + || next.kind(&conv) == T!['('] => { tt::Spacing::Alone } - Some(next) if next.kind().is_punct() && next.kind() != UNDERSCORE => { + Some(next) if next.kind(&conv).is_punct() && next.kind(&conv) != UNDERSCORE => { tt::Spacing::Joint } _ => tt::Spacing::Alone, }; - let char = match token.to_char() { + let char = match token.to_char(&conv) { Some(c) => c, None => { panic!("Token from lexer must be single char: token = {:#?}", token); @@ -222,7 +220,7 @@ fn convert_tokens<C: TokenConvertor>(conv: &mut C) -> tt::Subtree { } else { macro_rules! make_leaf { ($i:ident) => { - tt::$i { id: conv.id_alloc().alloc(range), text: token.to_text() }.into() + tt::$i { id: conv.id_alloc().alloc(range), text: token.to_text(conv) }.into() }; } let leaf: tt::Leaf = match k { @@ -243,7 +241,7 @@ fn convert_tokens<C: TokenConvertor>(conv: &mut C) -> tt::Subtree { let r = TextRange::at(range.start() + char_unit, range.len() - char_unit); let ident = tt::Leaf::from(tt::Ident { - text: SmolStr::new(&token.to_text()[1..]), + text: SmolStr::new(&token.to_text(conv)[1..]), id: conv.id_alloc().alloc(r), }); result.push(ident.into()); @@ -392,22 +390,21 @@ impl TokenIdAlloc { /// A Raw Token (straightly from lexer) convertor struct RawConvertor<'a> { - text: &'a str, - offset: TextSize, + lexed: parser::LexedStr<'a>, + pos: usize, id_alloc: TokenIdAlloc, - inner: std::slice::Iter<'a, RawToken>, } -trait SrcToken: std::fmt::Debug { - fn kind(&self) -> SyntaxKind; +trait SrcToken<Ctx>: std::fmt::Debug { + fn kind(&self, ctx: &Ctx) -> SyntaxKind; - fn to_char(&self) -> Option<char>; + fn to_char(&self, ctx: &Ctx) -> Option<char>; - fn to_text(&self) -> SmolStr; + fn to_text(&self, ctx: &Ctx) -> SmolStr; } -trait TokenConvertor { - type Token: SrcToken; +trait TokenConvertor: Sized { + type Token: SrcToken<Self>; fn convert_doc_comment(&self, token: &Self::Token) -> Option<Vec<tt::TokenTree>>; @@ -418,42 +415,45 @@ trait TokenConvertor { fn id_alloc(&mut self) -> &mut TokenIdAlloc; } -impl<'a> SrcToken for (&'a RawToken, &'a str) { - fn kind(&self) -> SyntaxKind { - self.0.kind +impl<'a> SrcToken<RawConvertor<'a>> for usize { + fn kind(&self, ctx: &RawConvertor<'a>) -> SyntaxKind { + ctx.lexed.kind(*self) } - fn to_char(&self) -> Option<char> { - self.1.chars().next() + fn to_char(&self, ctx: &RawConvertor<'a>) -> Option<char> { + ctx.lexed.text(*self).chars().next() } - fn to_text(&self) -> SmolStr { - self.1.into() + fn to_text(&self, ctx: &RawConvertor<'_>) -> SmolStr { + ctx.lexed.text(*self).into() } } impl<'a> TokenConvertor for RawConvertor<'a> { - type Token = (&'a RawToken, &'a str); + type Token = usize; - fn convert_doc_comment(&self, token: &Self::Token) -> Option<Vec<tt::TokenTree>> { - convert_doc_comment(&doc_comment(token.1)) + fn convert_doc_comment(&self, token: &usize) -> Option<Vec<tt::TokenTree>> { + let text = self.lexed.text(*token); + convert_doc_comment(&doc_comment(text)) } fn bump(&mut self) -> Option<(Self::Token, TextRange)> { - let token = self.inner.next()?; - let range = TextRange::at(self.offset, token.len); - self.offset += token.len; + if self.pos == self.lexed.len() { + return None; + } + let token = self.pos; + self.pos += 1; + let range = self.lexed.text_range(token); + let range = TextRange::new(range.start.try_into().unwrap(), range.end.try_into().unwrap()); - Some(((token, &self.text[range]), range)) + Some((token, range)) } fn peek(&self) -> Option<Self::Token> { - let token = self.inner.as_slice().get(0); - - token.map(|it| { - let range = TextRange::at(self.offset, it.len); - (it, &self.text[range]) - }) + if self.pos == self.lexed.len() { + return None; + } + Some(self.pos) } fn id_alloc(&mut self) -> &mut TokenIdAlloc { @@ -523,17 +523,17 @@ impl SynToken { } } -impl SrcToken for SynToken { - fn kind(&self) -> SyntaxKind { +impl<'a> SrcToken<Convertor<'a>> for SynToken { + fn kind(&self, _ctx: &Convertor<'a>) -> SyntaxKind { self.token().kind() } - fn to_char(&self) -> Option<char> { + fn to_char(&self, _ctx: &Convertor<'a>) -> Option<char> { match self { SynToken::Ordinary(_) => None, SynToken::Punch(it, i) => it.text().chars().nth((*i).into()), } } - fn to_text(&self) -> SmolStr { + fn to_text(&self, _ctx: &Convertor<'a>) -> SmolStr { self.token().text().into() } } diff --git a/crates/mbe/src/to_parser_tokens.rs b/crates/mbe/src/to_parser_tokens.rs index 644689f432..f419c78d46 100644 --- a/crates/mbe/src/to_parser_tokens.rs +++ b/crates/mbe/src/to_parser_tokens.rs @@ -1,7 +1,7 @@ //! Convert macro-by-example tokens which are specific to macro expansion into a //! format that works for our parser. -use syntax::{lex_single_syntax_kind, SyntaxKind, SyntaxKind::*, T}; +use syntax::{SyntaxKind, SyntaxKind::*, T}; use tt::buffer::TokenBuffer; pub(crate) fn to_parser_tokens(buffer: &TokenBuffer) -> parser::Tokens { @@ -35,7 +35,7 @@ pub(crate) fn to_parser_tokens(buffer: &TokenBuffer) -> parser::Tokens { let is_negated = lit.text.starts_with('-'); let inner_text = &lit.text[if is_negated { 1 } else { 0 }..]; - let kind = lex_single_syntax_kind(inner_text) + let kind = parser::LexedStr::single_token(inner_text) .map(|(kind, _error)| kind) .filter(|kind| { kind.is_literal() diff --git a/crates/parser/Cargo.toml b/crates/parser/Cargo.toml index d830ea93c8..4028082d08 100644 --- a/crates/parser/Cargo.toml +++ b/crates/parser/Cargo.toml @@ -11,5 +11,8 @@ doctest = false [dependencies] drop_bomb = "0.1.4" - +rustc_lexer = { version = "725.0.0", package = "rustc-ap-rustc_lexer" } limit = { path = "../limit", version = "0.0.0" } + +[dev-dependencies] +expect-test = "1.2" diff --git a/crates/parser/src/lexed_str.rs b/crates/parser/src/lexed_str.rs new file mode 100644 index 0000000000..b8936c3440 --- /dev/null +++ b/crates/parser/src/lexed_str.rs @@ -0,0 +1,300 @@ +//! Lexing `&str` into a sequence of Rust tokens. +//! +//! Note that strictly speaking the parser in this crate is not required to work +//! on tokens which originated from text. Macros, eg, can synthesize tokens out +//! of thin air. So, ideally, lexer should be an orthogonal crate. It is however +//! convenient to include a text-based lexer here! +//! +//! Note that these tokens, unlike the tokens we feed into the parser, do +//! include info about comments and whitespace. + +use std::ops; + +use crate::{ + SyntaxKind::{self, *}, + T, +}; + +pub struct LexedStr<'a> { + text: &'a str, + kind: Vec<SyntaxKind>, + start: Vec<u32>, + error: Vec<LexError>, +} + +struct LexError { + msg: String, + token: u32, +} + +impl<'a> LexedStr<'a> { + pub fn new(text: &'a str) -> LexedStr<'a> { + let mut res = LexedStr { text, kind: Vec::new(), start: Vec::new(), error: Vec::new() }; + + let mut offset = 0; + if let Some(shebang_len) = rustc_lexer::strip_shebang(text) { + res.push(SHEBANG, offset); + offset = shebang_len + }; + for token in rustc_lexer::tokenize(&text[offset..]) { + let token_text = &text[offset..][..token.len]; + + let (kind, err) = from_rustc(&token.kind, token_text); + res.push(kind, offset); + offset += token.len; + + if let Some(err) = err { + let token = res.len() as u32; + let msg = err.to_string(); + res.error.push(LexError { msg, token }); + } + } + res.push(EOF, offset); + + res + } + + pub fn single_token(text: &'a str) -> Option<(SyntaxKind, Option<String>)> { + if text.is_empty() { + return None; + } + + let token = rustc_lexer::first_token(text); + if token.len != text.len() { + return None; + } + + let (kind, err) = from_rustc(&token.kind, text); + Some((kind, err.map(|it| it.to_owned()))) + } + + pub fn as_str(&self) -> &str { + self.text + } + + pub fn len(&self) -> usize { + self.kind.len() - 1 + } + + pub fn is_empty(&self) -> bool { + self.len() == 0 + } + + pub fn kind(&self, i: usize) -> SyntaxKind { + assert!(i < self.len()); + self.kind[i] + } + + pub fn text(&self, i: usize) -> &str { + self.range_text(i..i + 1) + } + pub fn range_text(&self, r: ops::Range<usize>) -> &str { + assert!(r.start < r.end && r.end <= self.len()); + let lo = self.start[r.start] as usize; + let hi = self.start[r.end] as usize; + &self.text[lo..hi] + } + + // Naming is hard. + pub fn text_range(&self, i: usize) -> ops::Range<usize> { + assert!(i < self.len()); + let lo = self.start[i] as usize; + let hi = self.start[i + 1] as usize; + lo..hi + } + pub fn text_start(&self, i: usize) -> usize { + assert!(i <= self.len()); + self.start[i] as usize + } + pub fn text_len(&self, i: usize) -> usize { + assert!(i < self.len()); + let r = self.text_range(i); + r.end - r.start + } + + pub fn error(&self, i: usize) -> Option<&str> { + assert!(i < self.len()); + let err = self.error.binary_search_by_key(&(i as u32), |i| i.token).ok()?; + Some(self.error[err].msg.as_str()) + } + + pub fn errors(&self) -> impl Iterator<Item = (usize, &str)> + '_ { + self.error.iter().map(|it| (it.token as usize, it.msg.as_str())) + } + + pub fn to_tokens(&self) -> crate::Tokens { + let mut res = crate::Tokens::default(); + let mut was_joint = false; + for i in 0..self.len() { + let kind = self.kind(i); + if kind.is_trivia() { + was_joint = false + } else { + if kind == SyntaxKind::IDENT { + let token_text = self.text(i); + let contextual_kw = SyntaxKind::from_contextual_keyword(token_text) + .unwrap_or(SyntaxKind::IDENT); + res.push_ident(contextual_kw); + } else { + if was_joint { + res.was_joint(); + } + res.push(kind); + } + was_joint = true; + } + } + res + } + + fn push(&mut self, kind: SyntaxKind, offset: usize) { + self.kind.push(kind); + self.start.push(offset as u32); + } +} + +/// Returns `SyntaxKind` and an optional tokenize error message. +fn from_rustc( + kind: &rustc_lexer::TokenKind, + token_text: &str, +) -> (SyntaxKind, Option<&'static str>) { + // A note on an intended tradeoff: + // We drop some useful information here (see patterns with double dots `..`) + // Storing that info in `SyntaxKind` is not possible due to its layout requirements of + // being `u16` that come from `rowan::SyntaxKind`. + let mut err = ""; + + let syntax_kind = { + match kind { + rustc_lexer::TokenKind::LineComment { doc_style: _ } => COMMENT, + rustc_lexer::TokenKind::BlockComment { doc_style: _, terminated } => { + if !terminated { + err = "Missing trailing `*/` symbols to terminate the block comment"; + } + COMMENT + } + + rustc_lexer::TokenKind::Whitespace => WHITESPACE, + + rustc_lexer::TokenKind::Ident if token_text == "_" => UNDERSCORE, + rustc_lexer::TokenKind::Ident => SyntaxKind::from_keyword(token_text).unwrap_or(IDENT), + + rustc_lexer::TokenKind::RawIdent => IDENT, + rustc_lexer::TokenKind::Literal { kind, .. } => return from_rustc_literal(kind), + + rustc_lexer::TokenKind::Lifetime { starts_with_number } => { + if *starts_with_number { + err = "Lifetime name cannot start with a number"; + } + LIFETIME_IDENT + } + + rustc_lexer::TokenKind::Semi => T![;], + rustc_lexer::TokenKind::Comma => T![,], + rustc_lexer::TokenKind::Dot => T![.], + rustc_lexer::TokenKind::OpenParen => T!['('], + rustc_lexer::TokenKind::CloseParen => T![')'], + rustc_lexer::TokenKind::OpenBrace => T!['{'], + rustc_lexer::TokenKind::CloseBrace => T!['}'], + rustc_lexer::TokenKind::OpenBracket => T!['['], + rustc_lexer::TokenKind::CloseBracket => T![']'], + rustc_lexer::TokenKind::At => T![@], + rustc_lexer::TokenKind::Pound => T![#], + rustc_lexer::TokenKind::Tilde => T![~], + rustc_lexer::TokenKind::Question => T![?], + rustc_lexer::TokenKind::Colon => T![:], + rustc_lexer::TokenKind::Dollar => T![$], + rustc_lexer::TokenKind::Eq => T![=], + rustc_lexer::TokenKind::Bang => T![!], + rustc_lexer::TokenKind::Lt => T![<], + rustc_lexer::TokenKind::Gt => T![>], + rustc_lexer::TokenKind::Minus => T![-], + rustc_lexer::TokenKind::And => T![&], + rustc_lexer::TokenKind::Or => T![|], + rustc_lexer::TokenKind::Plus => T![+], + rustc_lexer::TokenKind::Star => T![*], + rustc_lexer::TokenKind::Slash => T![/], + rustc_lexer::TokenKind::Caret => T![^], + rustc_lexer::TokenKind::Percent => T![%], + rustc_lexer::TokenKind::Unknown => ERROR, + } + }; + + let err = if err.is_empty() { None } else { Some(err) }; + (syntax_kind, err) +} + +fn from_rustc_literal(kind: &rustc_lexer::LiteralKind) -> (SyntaxKind, Option<&'static str>) { + let mut err = ""; + + let syntax_kind = match *kind { + rustc_lexer::LiteralKind::Int { empty_int, base: _ } => { + if empty_int { + err = "Missing digits after the integer base prefix"; + } + INT_NUMBER + } + rustc_lexer::LiteralKind::Float { empty_exponent, base: _ } => { + if empty_exponent { + err = "Missing digits after the exponent symbol"; + } + FLOAT_NUMBER + } + rustc_lexer::LiteralKind::Char { terminated } => { + if !terminated { + err = "Missing trailing `'` symbol to terminate the character literal"; + } + CHAR + } + rustc_lexer::LiteralKind::Byte { terminated } => { + if !terminated { + err = "Missing trailing `'` symbol to terminate the byte literal"; + } + BYTE + } + rustc_lexer::LiteralKind::Str { terminated } => { + if !terminated { + err = "Missing trailing `\"` symbol to terminate the string literal"; + } + STRING + } + rustc_lexer::LiteralKind::ByteStr { terminated } => { + if !terminated { + err = "Missing trailing `\"` symbol to terminate the byte string literal"; + } + BYTE_STRING + } + rustc_lexer::LiteralKind::RawStr { err: raw_str_err, .. } => { + if let Some(raw_str_err) = raw_str_err { + err = match raw_str_err { + rustc_lexer::RawStrError::InvalidStarter { .. } => "Missing `\"` symbol after `#` symbols to begin the raw string literal", + rustc_lexer::RawStrError::NoTerminator { expected, found, .. } => if expected == found { + "Missing trailing `\"` to terminate the raw string literal" + } else { + "Missing trailing `\"` with `#` symbols to terminate the raw string literal" + }, + rustc_lexer::RawStrError::TooManyDelimiters { .. } => "Too many `#` symbols: raw strings may be delimited by up to 65535 `#` symbols", + }; + }; + STRING + } + rustc_lexer::LiteralKind::RawByteStr { err: raw_str_err, .. } => { + if let Some(raw_str_err) = raw_str_err { + err = match raw_str_err { + rustc_lexer::RawStrError::InvalidStarter { .. } => "Missing `\"` symbol after `#` symbols to begin the raw byte string literal", + rustc_lexer::RawStrError::NoTerminator { expected, found, .. } => if expected == found { + "Missing trailing `\"` to terminate the raw byte string literal" + } else { + "Missing trailing `\"` with `#` symbols to terminate the raw byte string literal" + }, + rustc_lexer::RawStrError::TooManyDelimiters { .. } => "Too many `#` symbols: raw byte strings may be delimited by up to 65535 `#` symbols", + }; + }; + + BYTE_STRING + } + }; + + let err = if err.is_empty() { None } else { Some(err) }; + (syntax_kind, err) +} diff --git a/crates/parser/src/lib.rs b/crates/parser/src/lib.rs index 2e2d96d027..dc02ae6e83 100644 --- a/crates/parser/src/lib.rs +++ b/crates/parser/src/lib.rs @@ -18,6 +18,7 @@ //! [`Parser`]: crate::parser::Parser #![allow(rustdoc::private_intra_doc_links)] +mod lexed_str; mod token_set; mod syntax_kind; mod event; @@ -25,9 +26,12 @@ mod parser; mod grammar; mod tokens; +#[cfg(test)] +mod tests; + pub(crate) use token_set::TokenSet; -pub use crate::{syntax_kind::SyntaxKind, tokens::Tokens}; +pub use crate::{lexed_str::LexedStr, syntax_kind::SyntaxKind, tokens::Tokens}; #[derive(Debug, Clone, PartialEq, Eq, Hash)] pub struct ParseError(pub Box<String>); diff --git a/crates/parser/src/tests.rs b/crates/parser/src/tests.rs new file mode 100644 index 0000000000..ebba992561 --- /dev/null +++ b/crates/parser/src/tests.rs @@ -0,0 +1,71 @@ +use std::{ + fmt::Write, + fs, + path::{Path, PathBuf}, +}; + +use expect_test::expect_file; + +use crate::LexedStr; + +#[test] +fn valid_lexes_input() { + for case in TestCase::list("lexer/ok") { + let actual = lex(&case.text); + expect_file![case.txt].assert_eq(&actual) + } +} + +#[test] +fn invalid_lexes_input() { + for case in TestCase::list("lexer/err") { + let actual = lex(&case.text); + expect_file![case.txt].assert_eq(&actual) + } +} + +fn lex(text: &str) -> String { + let lexed = LexedStr::new(text); + + let mut res = String::new(); + for i in 0..lexed.len() { + let kind = lexed.kind(i); + let text = lexed.text(i); + let error = lexed.error(i); + + let error = error.map(|err| format!(" error: {}", err)).unwrap_or_default(); + writeln!(res, "{:?} {:?}{}", kind, text, error).unwrap(); + } + res +} + +#[derive(PartialEq, Eq, PartialOrd, Ord)] +struct TestCase { + rs: PathBuf, + txt: PathBuf, + text: String, +} + +impl TestCase { + fn list(path: &'static str) -> Vec<TestCase> { + let crate_root_dir = Path::new(env!("CARGO_MANIFEST_DIR")); + let test_data_dir = crate_root_dir.join("test_data"); + let dir = test_data_dir.join(path); + + let mut res = Vec::new(); + let read_dir = fs::read_dir(&dir) + .unwrap_or_else(|err| panic!("can't `read_dir` {}: {}", dir.display(), err)); + for file in read_dir { + let file = file.unwrap(); + let path = file.path(); + if path.extension().unwrap_or_default() == "rs" { + let rs = path; + let txt = rs.with_extension("txt"); + let text = fs::read_to_string(&rs).unwrap(); + res.push(TestCase { rs, txt, text }); + } + } + res.sort(); + res + } +} diff --git a/crates/parser/src/tokens.rs b/crates/parser/src/tokens.rs index 1c0672492d..4fc2361add 100644 --- a/crates/parser/src/tokens.rs +++ b/crates/parser/src/tokens.rs @@ -1,7 +1,8 @@ //! Input for the parser -- a sequence of tokens. //! //! As of now, parser doesn't have access to the *text* of the tokens, and makes -//! decisions based solely on their classification. +//! decisions based solely on their classification. Unlike `LexerToken`, the +//! `Tokens` doesn't include whitespace and comments. use crate::SyntaxKind; diff --git a/crates/syntax/test_data/lexer/err/0056_empty_exponent.rs b/crates/parser/test_data/lexer/err/empty_exponent.rs index 286584c887..286584c887 100644 --- a/crates/syntax/test_data/lexer/err/0056_empty_exponent.rs +++ b/crates/parser/test_data/lexer/err/empty_exponent.rs diff --git a/crates/parser/test_data/lexer/err/empty_exponent.txt b/crates/parser/test_data/lexer/err/empty_exponent.txt new file mode 100644 index 0000000000..af03d73ced --- /dev/null +++ b/crates/parser/test_data/lexer/err/empty_exponent.txt @@ -0,0 +1,48 @@ +FLOAT_NUMBER "0e" error: Missing digits after the exponent symbol +WHITESPACE "\n" +FLOAT_NUMBER "0E" error: Missing digits after the exponent symbol +WHITESPACE "\n\n" +FLOAT_NUMBER "42e+" error: Missing digits after the exponent symbol +WHITESPACE "\n" +FLOAT_NUMBER "42e-" error: Missing digits after the exponent symbol +WHITESPACE "\n" +FLOAT_NUMBER "42E+" error: Missing digits after the exponent symbol +WHITESPACE "\n" +FLOAT_NUMBER "42E-" error: Missing digits after the exponent symbol +WHITESPACE "\n\n" +INT_NUMBER "42" +DOT "." +IDENT "e" +PLUS "+" +WHITESPACE "\n" +INT_NUMBER "42" +DOT "." +IDENT "e" +MINUS "-" +WHITESPACE "\n" +INT_NUMBER "42" +DOT "." +IDENT "E" +PLUS "+" +WHITESPACE "\n" +INT_NUMBER "42" +DOT "." +IDENT "E" +MINUS "-" +WHITESPACE "\n\n" +FLOAT_NUMBER "42.2e+" error: Missing digits after the exponent symbol +WHITESPACE "\n" +FLOAT_NUMBER "42.2e-" error: Missing digits after the exponent symbol +WHITESPACE "\n" +FLOAT_NUMBER "42.2E+" error: Missing digits after the exponent symbol +WHITESPACE "\n" +FLOAT_NUMBER "42.2E-" error: Missing digits after the exponent symbol +WHITESPACE "\n\n" +FLOAT_NUMBER "42.2e+f32" error: Missing digits after the exponent symbol +WHITESPACE "\n" +FLOAT_NUMBER "42.2e-f32" error: Missing digits after the exponent symbol +WHITESPACE "\n" +FLOAT_NUMBER "42.2E+f32" error: Missing digits after the exponent symbol +WHITESPACE "\n" +FLOAT_NUMBER "42.2E-f32" error: Missing digits after the exponent symbol +WHITESPACE "\n" diff --git a/crates/syntax/test_data/lexer/err/0055_empty_int.rs b/crates/parser/test_data/lexer/err/empty_int.rs index aa2a9fdca1..aa2a9fdca1 100644 --- a/crates/syntax/test_data/lexer/err/0055_empty_int.rs +++ b/crates/parser/test_data/lexer/err/empty_int.rs diff --git a/crates/parser/test_data/lexer/err/empty_int.txt b/crates/parser/test_data/lexer/err/empty_int.txt new file mode 100644 index 0000000000..7f7194f452 --- /dev/null +++ b/crates/parser/test_data/lexer/err/empty_int.txt @@ -0,0 +1,26 @@ +INT_NUMBER "0b" error: Missing digits after the integer base prefix +WHITESPACE "\n" +INT_NUMBER "0o" error: Missing digits after the integer base prefix +WHITESPACE "\n" +INT_NUMBER "0x" error: Missing digits after the integer base prefix +WHITESPACE "\n\n" +INT_NUMBER "0b_" error: Missing digits after the integer base prefix +WHITESPACE "\n" +INT_NUMBER "0o_" error: Missing digits after the integer base prefix +WHITESPACE "\n" +INT_NUMBER "0x_" error: Missing digits after the integer base prefix +WHITESPACE "\n\n" +INT_NUMBER "0bnoDigit" error: Missing digits after the integer base prefix +WHITESPACE "\n" +INT_NUMBER "0onoDigit" error: Missing digits after the integer base prefix +WHITESPACE "\n" +INT_NUMBER "0xnoDigit" error: Missing digits after the integer base prefix +WHITESPACE "\n\n" +INT_NUMBER "0xG" error: Missing digits after the integer base prefix +WHITESPACE "\n" +INT_NUMBER "0xg" error: Missing digits after the integer base prefix +WHITESPACE "\n\n" +INT_NUMBER "0x_g" error: Missing digits after the integer base prefix +WHITESPACE "\n" +INT_NUMBER "0x_G" error: Missing digits after the integer base prefix +WHITESPACE "\n" diff --git a/crates/syntax/test_data/lexer/err/0057_lifetime_starts_with_a_number.rs b/crates/parser/test_data/lexer/err/lifetime_starts_with_a_number.rs index a7698a404a..a7698a404a 100644 --- a/crates/syntax/test_data/lexer/err/0057_lifetime_starts_with_a_number.rs +++ b/crates/parser/test_data/lexer/err/lifetime_starts_with_a_number.rs diff --git a/crates/parser/test_data/lexer/err/lifetime_starts_with_a_number.txt b/crates/parser/test_data/lexer/err/lifetime_starts_with_a_number.txt new file mode 100644 index 0000000000..e919bf2a4a --- /dev/null +++ b/crates/parser/test_data/lexer/err/lifetime_starts_with_a_number.txt @@ -0,0 +1,4 @@ +LIFETIME_IDENT "'1" error: Lifetime name cannot start with a number +WHITESPACE "\n" +LIFETIME_IDENT "'1lifetime" error: Lifetime name cannot start with a number +WHITESPACE "\n" diff --git a/crates/syntax/test_data/lexer/err/0051_unclosed_block_comment_at_eof.rs b/crates/parser/test_data/lexer/err/unclosed_block_comment_at_eof.rs index 22e83649f7..22e83649f7 100644 --- a/crates/syntax/test_data/lexer/err/0051_unclosed_block_comment_at_eof.rs +++ b/crates/parser/test_data/lexer/err/unclosed_block_comment_at_eof.rs diff --git a/crates/parser/test_data/lexer/err/unclosed_block_comment_at_eof.txt b/crates/parser/test_data/lexer/err/unclosed_block_comment_at_eof.txt new file mode 100644 index 0000000000..7d2c329762 --- /dev/null +++ b/crates/parser/test_data/lexer/err/unclosed_block_comment_at_eof.txt @@ -0,0 +1 @@ +COMMENT "/*" error: Missing trailing `*/` symbols to terminate the block comment diff --git a/crates/syntax/test_data/lexer/err/0052_unclosed_block_comment_with_content.rs b/crates/parser/test_data/lexer/err/unclosed_block_comment_with_content.rs index c45c2844db..c45c2844db 100644 --- a/crates/syntax/test_data/lexer/err/0052_unclosed_block_comment_with_content.rs +++ b/crates/parser/test_data/lexer/err/unclosed_block_comment_with_content.rs diff --git a/crates/parser/test_data/lexer/err/unclosed_block_comment_with_content.txt b/crates/parser/test_data/lexer/err/unclosed_block_comment_with_content.txt new file mode 100644 index 0000000000..227a20660f --- /dev/null +++ b/crates/parser/test_data/lexer/err/unclosed_block_comment_with_content.txt @@ -0,0 +1 @@ +COMMENT "/* comment\n" error: Missing trailing `*/` symbols to terminate the block comment diff --git a/crates/syntax/test_data/lexer/err/0009_unclosed_byte_at_eof.rs b/crates/parser/test_data/lexer/err/unclosed_byte_at_eof.rs index 795dc7e25c..795dc7e25c 100644 --- a/crates/syntax/test_data/lexer/err/0009_unclosed_byte_at_eof.rs +++ b/crates/parser/test_data/lexer/err/unclosed_byte_at_eof.rs diff --git a/crates/parser/test_data/lexer/err/unclosed_byte_at_eof.txt b/crates/parser/test_data/lexer/err/unclosed_byte_at_eof.txt new file mode 100644 index 0000000000..36944dbb2d --- /dev/null +++ b/crates/parser/test_data/lexer/err/unclosed_byte_at_eof.txt @@ -0,0 +1 @@ +BYTE "b'" error: Missing trailing `'` symbol to terminate the byte literal diff --git a/crates/syntax/test_data/lexer/err/0025_unclosed_byte_string_at_eof.rs b/crates/parser/test_data/lexer/err/unclosed_byte_string_at_eof.rs index 36f4f43218..36f4f43218 100644 --- a/crates/syntax/test_data/lexer/err/0025_unclosed_byte_string_at_eof.rs +++ b/crates/parser/test_data/lexer/err/unclosed_byte_string_at_eof.rs diff --git a/crates/parser/test_data/lexer/err/unclosed_byte_string_at_eof.txt b/crates/parser/test_data/lexer/err/unclosed_byte_string_at_eof.txt new file mode 100644 index 0000000000..534a3cadcc --- /dev/null +++ b/crates/parser/test_data/lexer/err/unclosed_byte_string_at_eof.txt @@ -0,0 +1 @@ +BYTE_STRING "b\"" error: Missing trailing `"` symbol to terminate the byte string literal diff --git a/crates/syntax/test_data/lexer/err/0027_unclosed_byte_string_with_ascii_escape.rs b/crates/parser/test_data/lexer/err/unclosed_byte_string_with_ascii_escape.rs index 836c112c15..836c112c15 100644 --- a/crates/syntax/test_data/lexer/err/0027_unclosed_byte_string_with_ascii_escape.rs +++ b/crates/parser/test_data/lexer/err/unclosed_byte_string_with_ascii_escape.rs diff --git a/crates/parser/test_data/lexer/err/unclosed_byte_string_with_ascii_escape.txt b/crates/parser/test_data/lexer/err/unclosed_byte_string_with_ascii_escape.txt new file mode 100644 index 0000000000..03f61de9a8 --- /dev/null +++ b/crates/parser/test_data/lexer/err/unclosed_byte_string_with_ascii_escape.txt @@ -0,0 +1 @@ +BYTE_STRING "b\"\\x7f" error: Missing trailing `"` symbol to terminate the byte string literal diff --git a/crates/syntax/test_data/lexer/err/0026_unclosed_byte_string_with_ferris.rs b/crates/parser/test_data/lexer/err/unclosed_byte_string_with_ferris.rs index 3c23a03722..3c23a03722 100644 --- a/crates/syntax/test_data/lexer/err/0026_unclosed_byte_string_with_ferris.rs +++ b/crates/parser/test_data/lexer/err/unclosed_byte_string_with_ferris.rs diff --git a/crates/parser/test_data/lexer/err/unclosed_byte_string_with_ferris.txt b/crates/parser/test_data/lexer/err/unclosed_byte_string_with_ferris.txt new file mode 100644 index 0000000000..e11d49d1ee --- /dev/null +++ b/crates/parser/test_data/lexer/err/unclosed_byte_string_with_ferris.txt @@ -0,0 +1 @@ +BYTE_STRING "b\"π¦" error: Missing trailing `"` symbol to terminate the byte string literal diff --git a/crates/syntax/test_data/lexer/err/0030_unclosed_byte_string_with_slash.rs b/crates/parser/test_data/lexer/err/unclosed_byte_string_with_slash.rs index cce6615381..cce6615381 100644 --- a/crates/syntax/test_data/lexer/err/0030_unclosed_byte_string_with_slash.rs +++ b/crates/parser/test_data/lexer/err/unclosed_byte_string_with_slash.rs diff --git a/crates/parser/test_data/lexer/err/unclosed_byte_string_with_slash.txt b/crates/parser/test_data/lexer/err/unclosed_byte_string_with_slash.txt new file mode 100644 index 0000000000..4e374b1206 --- /dev/null +++ b/crates/parser/test_data/lexer/err/unclosed_byte_string_with_slash.txt @@ -0,0 +1 @@ +BYTE_STRING "b\"\\" error: Missing trailing `"` symbol to terminate the byte string literal diff --git a/crates/syntax/test_data/lexer/err/0032_unclosed_byte_string_with_slash_double_quote.rs b/crates/parser/test_data/lexer/err/unclosed_byte_string_with_slash_double_quote.rs index f2ff58ba9a..f2ff58ba9a 100644 --- a/crates/syntax/test_data/lexer/err/0032_unclosed_byte_string_with_slash_double_quote.rs +++ b/crates/parser/test_data/lexer/err/unclosed_byte_string_with_slash_double_quote.rs diff --git a/crates/parser/test_data/lexer/err/unclosed_byte_string_with_slash_double_quote.txt b/crates/parser/test_data/lexer/err/unclosed_byte_string_with_slash_double_quote.txt new file mode 100644 index 0000000000..ee19975860 --- /dev/null +++ b/crates/parser/test_data/lexer/err/unclosed_byte_string_with_slash_double_quote.txt @@ -0,0 +1 @@ +BYTE_STRING "b\"\\\"" error: Missing trailing `"` symbol to terminate the byte string literal diff --git a/crates/syntax/test_data/lexer/err/0031_unclosed_byte_string_with_slash_n.rs b/crates/parser/test_data/lexer/err/unclosed_byte_string_with_slash_n.rs index 5e680aabb7..5e680aabb7 100644 --- a/crates/syntax/test_data/lexer/err/0031_unclosed_byte_string_with_slash_n.rs +++ b/crates/parser/test_data/lexer/err/unclosed_byte_string_with_slash_n.rs diff --git a/crates/parser/test_data/lexer/err/unclosed_byte_string_with_slash_n.txt b/crates/parser/test_data/lexer/err/unclosed_byte_string_with_slash_n.txt new file mode 100644 index 0000000000..b109d8629c --- /dev/null +++ b/crates/parser/test_data/lexer/err/unclosed_byte_string_with_slash_n.txt @@ -0,0 +1 @@ +BYTE_STRING "b\"\\n" error: Missing trailing `"` symbol to terminate the byte string literal diff --git a/crates/syntax/test_data/lexer/err/0029_unclosed_byte_string_with_space.rs b/crates/parser/test_data/lexer/err/unclosed_byte_string_with_space.rs index d6898541e6..d6898541e6 100644 --- a/crates/syntax/test_data/lexer/err/0029_unclosed_byte_string_with_space.rs +++ b/crates/parser/test_data/lexer/err/unclosed_byte_string_with_space.rs diff --git a/crates/parser/test_data/lexer/err/unclosed_byte_string_with_space.txt b/crates/parser/test_data/lexer/err/unclosed_byte_string_with_space.txt new file mode 100644 index 0000000000..eaca94fa41 --- /dev/null +++ b/crates/parser/test_data/lexer/err/unclosed_byte_string_with_space.txt @@ -0,0 +1 @@ +BYTE_STRING "b\" " error: Missing trailing `"` symbol to terminate the byte string literal diff --git a/crates/syntax/test_data/lexer/err/0028_unclosed_byte_string_with_unicode_escape.rs b/crates/parser/test_data/lexer/err/unclosed_byte_string_with_unicode_escape.rs index 1c6df1d00e..1c6df1d00e 100644 --- a/crates/syntax/test_data/lexer/err/0028_unclosed_byte_string_with_unicode_escape.rs +++ b/crates/parser/test_data/lexer/err/unclosed_byte_string_with_unicode_escape.rs diff --git a/crates/parser/test_data/lexer/err/unclosed_byte_string_with_unicode_escape.txt b/crates/parser/test_data/lexer/err/unclosed_byte_string_with_unicode_escape.txt new file mode 100644 index 0000000000..3b79f48bcd --- /dev/null +++ b/crates/parser/test_data/lexer/err/unclosed_byte_string_with_unicode_escape.txt @@ -0,0 +1 @@ +BYTE_STRING "b\"\\u{20AA}" error: Missing trailing `"` symbol to terminate the byte string literal diff --git a/crates/syntax/test_data/lexer/err/0011_unclosed_byte_with_ascii_escape.rs b/crates/parser/test_data/lexer/err/unclosed_byte_with_ascii_escape.rs index d146a8090d..d146a8090d 100644 --- a/crates/syntax/test_data/lexer/err/0011_unclosed_byte_with_ascii_escape.rs +++ b/crates/parser/test_data/lexer/err/unclosed_byte_with_ascii_escape.rs diff --git a/crates/parser/test_data/lexer/err/unclosed_byte_with_ascii_escape.txt b/crates/parser/test_data/lexer/err/unclosed_byte_with_ascii_escape.txt new file mode 100644 index 0000000000..5525376f45 --- /dev/null +++ b/crates/parser/test_data/lexer/err/unclosed_byte_with_ascii_escape.txt @@ -0,0 +1 @@ +BYTE "b'\\x7f" error: Missing trailing `'` symbol to terminate the byte literal diff --git a/crates/syntax/test_data/lexer/err/0010_unclosed_byte_with_ferris.rs b/crates/parser/test_data/lexer/err/unclosed_byte_with_ferris.rs index c9230dc24e..c9230dc24e 100644 --- a/crates/syntax/test_data/lexer/err/0010_unclosed_byte_with_ferris.rs +++ b/crates/parser/test_data/lexer/err/unclosed_byte_with_ferris.rs diff --git a/crates/parser/test_data/lexer/err/unclosed_byte_with_ferris.txt b/crates/parser/test_data/lexer/err/unclosed_byte_with_ferris.txt new file mode 100644 index 0000000000..e7a8be4f6e --- /dev/null +++ b/crates/parser/test_data/lexer/err/unclosed_byte_with_ferris.txt @@ -0,0 +1 @@ +BYTE "b'π¦" error: Missing trailing `'` symbol to terminate the byte literal diff --git a/crates/syntax/test_data/lexer/err/0014_unclosed_byte_with_slash.rs b/crates/parser/test_data/lexer/err/unclosed_byte_with_slash.rs index abffa5037c..abffa5037c 100644 --- a/crates/syntax/test_data/lexer/err/0014_unclosed_byte_with_slash.rs +++ b/crates/parser/test_data/lexer/err/unclosed_byte_with_slash.rs diff --git a/crates/parser/test_data/lexer/err/unclosed_byte_with_slash.txt b/crates/parser/test_data/lexer/err/unclosed_byte_with_slash.txt new file mode 100644 index 0000000000..d9937135a9 --- /dev/null +++ b/crates/parser/test_data/lexer/err/unclosed_byte_with_slash.txt @@ -0,0 +1 @@ +BYTE "b'\\" error: Missing trailing `'` symbol to terminate the byte literal diff --git a/crates/syntax/test_data/lexer/err/0015_unclosed_byte_with_slash_n.rs b/crates/parser/test_data/lexer/err/unclosed_byte_with_slash_n.rs index 4f46836a93..4f46836a93 100644 --- a/crates/syntax/test_data/lexer/err/0015_unclosed_byte_with_slash_n.rs +++ b/crates/parser/test_data/lexer/err/unclosed_byte_with_slash_n.rs diff --git a/crates/parser/test_data/lexer/err/unclosed_byte_with_slash_n.txt b/crates/parser/test_data/lexer/err/unclosed_byte_with_slash_n.txt new file mode 100644 index 0000000000..c408cdb2b5 --- /dev/null +++ b/crates/parser/test_data/lexer/err/unclosed_byte_with_slash_n.txt @@ -0,0 +1 @@ +BYTE "b'\\n" error: Missing trailing `'` symbol to terminate the byte literal diff --git a/crates/syntax/test_data/lexer/err/0016_unclosed_byte_with_slash_single_quote.rs b/crates/parser/test_data/lexer/err/unclosed_byte_with_slash_single_quote.rs index 645b641eed..645b641eed 100644 --- a/crates/syntax/test_data/lexer/err/0016_unclosed_byte_with_slash_single_quote.rs +++ b/crates/parser/test_data/lexer/err/unclosed_byte_with_slash_single_quote.rs diff --git a/crates/parser/test_data/lexer/err/unclosed_byte_with_slash_single_quote.txt b/crates/parser/test_data/lexer/err/unclosed_byte_with_slash_single_quote.txt new file mode 100644 index 0000000000..b331f95607 --- /dev/null +++ b/crates/parser/test_data/lexer/err/unclosed_byte_with_slash_single_quote.txt @@ -0,0 +1 @@ +BYTE "b'\\'" error: Missing trailing `'` symbol to terminate the byte literal diff --git a/crates/syntax/test_data/lexer/err/0013_unclosed_byte_with_space.rs b/crates/parser/test_data/lexer/err/unclosed_byte_with_space.rs index 93b7f9c87c..93b7f9c87c 100644 --- a/crates/syntax/test_data/lexer/err/0013_unclosed_byte_with_space.rs +++ b/crates/parser/test_data/lexer/err/unclosed_byte_with_space.rs diff --git a/crates/parser/test_data/lexer/err/unclosed_byte_with_space.txt b/crates/parser/test_data/lexer/err/unclosed_byte_with_space.txt new file mode 100644 index 0000000000..80c0e1c00a --- /dev/null +++ b/crates/parser/test_data/lexer/err/unclosed_byte_with_space.txt @@ -0,0 +1 @@ +BYTE "b' " error: Missing trailing `'` symbol to terminate the byte literal diff --git a/crates/syntax/test_data/lexer/err/0012_unclosed_byte_with_unicode_escape.rs b/crates/parser/test_data/lexer/err/unclosed_byte_with_unicode_escape.rs index a3dec7c255..a3dec7c255 100644 --- a/crates/syntax/test_data/lexer/err/0012_unclosed_byte_with_unicode_escape.rs +++ b/crates/parser/test_data/lexer/err/unclosed_byte_with_unicode_escape.rs diff --git a/crates/parser/test_data/lexer/err/unclosed_byte_with_unicode_escape.txt b/crates/parser/test_data/lexer/err/unclosed_byte_with_unicode_escape.txt new file mode 100644 index 0000000000..e1c3dc141e --- /dev/null +++ b/crates/parser/test_data/lexer/err/unclosed_byte_with_unicode_escape.txt @@ -0,0 +1 @@ +BYTE "b'\\u{20AA}" error: Missing trailing `'` symbol to terminate the byte literal diff --git a/crates/syntax/test_data/lexer/err/0001_unclosed_char_at_eof.rs b/crates/parser/test_data/lexer/err/unclosed_char_at_eof.rs index ad2823b48f..ad2823b48f 100644 --- a/crates/syntax/test_data/lexer/err/0001_unclosed_char_at_eof.rs +++ b/crates/parser/test_data/lexer/err/unclosed_char_at_eof.rs diff --git a/crates/parser/test_data/lexer/err/unclosed_char_at_eof.txt b/crates/parser/test_data/lexer/err/unclosed_char_at_eof.txt new file mode 100644 index 0000000000..218c7a2d76 --- /dev/null +++ b/crates/parser/test_data/lexer/err/unclosed_char_at_eof.txt @@ -0,0 +1 @@ +CHAR "'" error: Missing trailing `'` symbol to terminate the character literal diff --git a/crates/syntax/test_data/lexer/err/0003_unclosed_char_with_ascii_escape.rs b/crates/parser/test_data/lexer/err/unclosed_char_with_ascii_escape.rs index cf74b4dad3..cf74b4dad3 100644 --- a/crates/syntax/test_data/lexer/err/0003_unclosed_char_with_ascii_escape.rs +++ b/crates/parser/test_data/lexer/err/unclosed_char_with_ascii_escape.rs diff --git a/crates/parser/test_data/lexer/err/unclosed_char_with_ascii_escape.txt b/crates/parser/test_data/lexer/err/unclosed_char_with_ascii_escape.txt new file mode 100644 index 0000000000..a0d8e1b83a --- /dev/null +++ b/crates/parser/test_data/lexer/err/unclosed_char_with_ascii_escape.txt @@ -0,0 +1 @@ +CHAR "'\\x7f" error: Missing trailing `'` symbol to terminate the character literal diff --git a/crates/syntax/test_data/lexer/err/0002_unclosed_char_with_ferris.rs b/crates/parser/test_data/lexer/err/unclosed_char_with_ferris.rs index e264a41528..e264a41528 100644 --- a/crates/syntax/test_data/lexer/err/0002_unclosed_char_with_ferris.rs +++ b/crates/parser/test_data/lexer/err/unclosed_char_with_ferris.rs diff --git a/crates/parser/test_data/lexer/err/unclosed_char_with_ferris.txt b/crates/parser/test_data/lexer/err/unclosed_char_with_ferris.txt new file mode 100644 index 0000000000..56f19cce07 --- /dev/null +++ b/crates/parser/test_data/lexer/err/unclosed_char_with_ferris.txt @@ -0,0 +1 @@ +CHAR "'π¦" error: Missing trailing `'` symbol to terminate the character literal diff --git a/crates/syntax/test_data/lexer/err/0006_unclosed_char_with_slash.rs b/crates/parser/test_data/lexer/err/unclosed_char_with_slash.rs index 6ba258b109..6ba258b109 100644 --- a/crates/syntax/test_data/lexer/err/0006_unclosed_char_with_slash.rs +++ b/crates/parser/test_data/lexer/err/unclosed_char_with_slash.rs diff --git a/crates/parser/test_data/lexer/err/unclosed_char_with_slash.txt b/crates/parser/test_data/lexer/err/unclosed_char_with_slash.txt new file mode 100644 index 0000000000..cfa0e0752a --- /dev/null +++ b/crates/parser/test_data/lexer/err/unclosed_char_with_slash.txt @@ -0,0 +1 @@ +CHAR "'\\" error: Missing trailing `'` symbol to terminate the character literal diff --git a/crates/syntax/test_data/lexer/err/0007_unclosed_char_with_slash_n.rs b/crates/parser/test_data/lexer/err/unclosed_char_with_slash_n.rs index 78bef7e3eb..78bef7e3eb 100644 --- a/crates/syntax/test_data/lexer/err/0007_unclosed_char_with_slash_n.rs +++ b/crates/parser/test_data/lexer/err/unclosed_char_with_slash_n.rs diff --git a/crates/parser/test_data/lexer/err/unclosed_char_with_slash_n.txt b/crates/parser/test_data/lexer/err/unclosed_char_with_slash_n.txt new file mode 100644 index 0000000000..6a42a4e22b --- /dev/null +++ b/crates/parser/test_data/lexer/err/unclosed_char_with_slash_n.txt @@ -0,0 +1 @@ +CHAR "'\\n" error: Missing trailing `'` symbol to terminate the character literal diff --git a/crates/syntax/test_data/lexer/err/0008_unclosed_char_with_slash_single_quote.rs b/crates/parser/test_data/lexer/err/unclosed_char_with_slash_single_quote.rs index a0e722065b..a0e722065b 100644 --- a/crates/syntax/test_data/lexer/err/0008_unclosed_char_with_slash_single_quote.rs +++ b/crates/parser/test_data/lexer/err/unclosed_char_with_slash_single_quote.rs diff --git a/crates/parser/test_data/lexer/err/unclosed_char_with_slash_single_quote.txt b/crates/parser/test_data/lexer/err/unclosed_char_with_slash_single_quote.txt new file mode 100644 index 0000000000..1275f6aa85 --- /dev/null +++ b/crates/parser/test_data/lexer/err/unclosed_char_with_slash_single_quote.txt @@ -0,0 +1 @@ +CHAR "'\\'" error: Missing trailing `'` symbol to terminate the character literal diff --git a/crates/syntax/test_data/lexer/err/0005_unclosed_char_with_space.rs b/crates/parser/test_data/lexer/err/unclosed_char_with_space.rs index 309ecfe478..309ecfe478 100644 --- a/crates/syntax/test_data/lexer/err/0005_unclosed_char_with_space.rs +++ b/crates/parser/test_data/lexer/err/unclosed_char_with_space.rs diff --git a/crates/parser/test_data/lexer/err/unclosed_char_with_space.txt b/crates/parser/test_data/lexer/err/unclosed_char_with_space.txt new file mode 100644 index 0000000000..746c425c4e --- /dev/null +++ b/crates/parser/test_data/lexer/err/unclosed_char_with_space.txt @@ -0,0 +1 @@ +CHAR "' " error: Missing trailing `'` symbol to terminate the character literal diff --git a/crates/syntax/test_data/lexer/err/0004_unclosed_char_with_unicode_escape.rs b/crates/parser/test_data/lexer/err/unclosed_char_with_unicode_escape.rs index 50be91f685..50be91f685 100644 --- a/crates/syntax/test_data/lexer/err/0004_unclosed_char_with_unicode_escape.rs +++ b/crates/parser/test_data/lexer/err/unclosed_char_with_unicode_escape.rs diff --git a/crates/parser/test_data/lexer/err/unclosed_char_with_unicode_escape.txt b/crates/parser/test_data/lexer/err/unclosed_char_with_unicode_escape.txt new file mode 100644 index 0000000000..9abd590982 --- /dev/null +++ b/crates/parser/test_data/lexer/err/unclosed_char_with_unicode_escape.txt @@ -0,0 +1 @@ +CHAR "'\\u{20AA}" error: Missing trailing `'` symbol to terminate the character literal diff --git a/crates/syntax/test_data/lexer/err/0053_unclosed_nested_block_comment_entirely.rs b/crates/parser/test_data/lexer/err/unclosed_nested_block_comment_entirely.rs index 3fcfc96600..3fcfc96600 100644 --- a/crates/syntax/test_data/lexer/err/0053_unclosed_nested_block_comment_entirely.rs +++ b/crates/parser/test_data/lexer/err/unclosed_nested_block_comment_entirely.rs diff --git a/crates/parser/test_data/lexer/err/unclosed_nested_block_comment_entirely.txt b/crates/parser/test_data/lexer/err/unclosed_nested_block_comment_entirely.txt new file mode 100644 index 0000000000..15ce8905a9 --- /dev/null +++ b/crates/parser/test_data/lexer/err/unclosed_nested_block_comment_entirely.txt @@ -0,0 +1 @@ +COMMENT "/* /* /*\n" error: Missing trailing `*/` symbols to terminate the block comment diff --git a/crates/syntax/test_data/lexer/err/0054_unclosed_nested_block_comment_partially.rs b/crates/parser/test_data/lexer/err/unclosed_nested_block_comment_partially.rs index 26c898f019..26c898f019 100644 --- a/crates/syntax/test_data/lexer/err/0054_unclosed_nested_block_comment_partially.rs +++ b/crates/parser/test_data/lexer/err/unclosed_nested_block_comment_partially.rs diff --git a/crates/parser/test_data/lexer/err/unclosed_nested_block_comment_partially.txt b/crates/parser/test_data/lexer/err/unclosed_nested_block_comment_partially.txt new file mode 100644 index 0000000000..e9b74ee7f8 --- /dev/null +++ b/crates/parser/test_data/lexer/err/unclosed_nested_block_comment_partially.txt @@ -0,0 +1 @@ +COMMENT "/** /*! /* comment */ */\n" error: Missing trailing `*/` symbols to terminate the block comment diff --git a/crates/syntax/test_data/lexer/err/0040_unclosed_raw_byte_string_at_eof.rs b/crates/parser/test_data/lexer/err/unclosed_raw_byte_string_at_eof.rs index ae5bae6223..ae5bae6223 100644 --- a/crates/syntax/test_data/lexer/err/0040_unclosed_raw_byte_string_at_eof.rs +++ b/crates/parser/test_data/lexer/err/unclosed_raw_byte_string_at_eof.rs diff --git a/crates/parser/test_data/lexer/err/unclosed_raw_byte_string_at_eof.txt b/crates/parser/test_data/lexer/err/unclosed_raw_byte_string_at_eof.txt new file mode 100644 index 0000000000..6ec1780c30 --- /dev/null +++ b/crates/parser/test_data/lexer/err/unclosed_raw_byte_string_at_eof.txt @@ -0,0 +1 @@ +BYTE_STRING "br##\"" error: Missing trailing `"` with `#` symbols to terminate the raw byte string literal diff --git a/crates/syntax/test_data/lexer/err/0042_unclosed_raw_byte_string_with_ascii_escape.rs b/crates/parser/test_data/lexer/err/unclosed_raw_byte_string_with_ascii_escape.rs index d50270afe1..d50270afe1 100644 --- a/crates/syntax/test_data/lexer/err/0042_unclosed_raw_byte_string_with_ascii_escape.rs +++ b/crates/parser/test_data/lexer/err/unclosed_raw_byte_string_with_ascii_escape.rs diff --git a/crates/parser/test_data/lexer/err/unclosed_raw_byte_string_with_ascii_escape.txt b/crates/parser/test_data/lexer/err/unclosed_raw_byte_string_with_ascii_escape.txt new file mode 100644 index 0000000000..d65f1bb2ff --- /dev/null +++ b/crates/parser/test_data/lexer/err/unclosed_raw_byte_string_with_ascii_escape.txt @@ -0,0 +1 @@ +BYTE_STRING "br##\"\\x7f" error: Missing trailing `"` with `#` symbols to terminate the raw byte string literal diff --git a/crates/syntax/test_data/lexer/err/0041_unclosed_raw_byte_string_with_ferris.rs b/crates/parser/test_data/lexer/err/unclosed_raw_byte_string_with_ferris.rs index 9ef01207a1..9ef01207a1 100644 --- a/crates/syntax/test_data/lexer/err/0041_unclosed_raw_byte_string_with_ferris.rs +++ b/crates/parser/test_data/lexer/err/unclosed_raw_byte_string_with_ferris.rs diff --git a/crates/parser/test_data/lexer/err/unclosed_raw_byte_string_with_ferris.txt b/crates/parser/test_data/lexer/err/unclosed_raw_byte_string_with_ferris.txt new file mode 100644 index 0000000000..0f9e0a1657 --- /dev/null +++ b/crates/parser/test_data/lexer/err/unclosed_raw_byte_string_with_ferris.txt @@ -0,0 +1 @@ +BYTE_STRING "br##\"π¦" error: Missing trailing `"` with `#` symbols to terminate the raw byte string literal diff --git a/crates/syntax/test_data/lexer/err/0045_unclosed_raw_byte_string_with_slash.rs b/crates/parser/test_data/lexer/err/unclosed_raw_byte_string_with_slash.rs index 0b3c015d74..0b3c015d74 100644 --- a/crates/syntax/test_data/lexer/err/0045_unclosed_raw_byte_string_with_slash.rs +++ b/crates/parser/test_data/lexer/err/unclosed_raw_byte_string_with_slash.rs diff --git a/crates/parser/test_data/lexer/err/unclosed_raw_byte_string_with_slash.txt b/crates/parser/test_data/lexer/err/unclosed_raw_byte_string_with_slash.txt new file mode 100644 index 0000000000..202dcd2d43 --- /dev/null +++ b/crates/parser/test_data/lexer/err/unclosed_raw_byte_string_with_slash.txt @@ -0,0 +1 @@ +BYTE_STRING "br##\"\\" error: Missing trailing `"` with `#` symbols to terminate the raw byte string literal diff --git a/crates/syntax/test_data/lexer/err/0046_unclosed_raw_byte_string_with_slash_n.rs b/crates/parser/test_data/lexer/err/unclosed_raw_byte_string_with_slash_n.rs index 0d8b0e7ab0..0d8b0e7ab0 100644 --- a/crates/syntax/test_data/lexer/err/0046_unclosed_raw_byte_string_with_slash_n.rs +++ b/crates/parser/test_data/lexer/err/unclosed_raw_byte_string_with_slash_n.rs diff --git a/crates/parser/test_data/lexer/err/unclosed_raw_byte_string_with_slash_n.txt b/crates/parser/test_data/lexer/err/unclosed_raw_byte_string_with_slash_n.txt new file mode 100644 index 0000000000..d45485b529 --- /dev/null +++ b/crates/parser/test_data/lexer/err/unclosed_raw_byte_string_with_slash_n.txt @@ -0,0 +1 @@ +BYTE_STRING "br##\"\\n" error: Missing trailing `"` with `#` symbols to terminate the raw byte string literal diff --git a/crates/syntax/test_data/lexer/err/0044_unclosed_raw_byte_string_with_space.rs b/crates/parser/test_data/lexer/err/unclosed_raw_byte_string_with_space.rs index 14c602fd2b..14c602fd2b 100644 --- a/crates/syntax/test_data/lexer/err/0044_unclosed_raw_byte_string_with_space.rs +++ b/crates/parser/test_data/lexer/err/unclosed_raw_byte_string_with_space.rs diff --git a/crates/parser/test_data/lexer/err/unclosed_raw_byte_string_with_space.txt b/crates/parser/test_data/lexer/err/unclosed_raw_byte_string_with_space.txt new file mode 100644 index 0000000000..1bfabbc3ab --- /dev/null +++ b/crates/parser/test_data/lexer/err/unclosed_raw_byte_string_with_space.txt @@ -0,0 +1 @@ +BYTE_STRING "br##\" " error: Missing trailing `"` with `#` symbols to terminate the raw byte string literal diff --git a/crates/syntax/test_data/lexer/err/0043_unclosed_raw_byte_string_with_unicode_escape.rs b/crates/parser/test_data/lexer/err/unclosed_raw_byte_string_with_unicode_escape.rs index 90e299a1a9..90e299a1a9 100644 --- a/crates/syntax/test_data/lexer/err/0043_unclosed_raw_byte_string_with_unicode_escape.rs +++ b/crates/parser/test_data/lexer/err/unclosed_raw_byte_string_with_unicode_escape.rs diff --git a/crates/parser/test_data/lexer/err/unclosed_raw_byte_string_with_unicode_escape.txt b/crates/parser/test_data/lexer/err/unclosed_raw_byte_string_with_unicode_escape.txt new file mode 100644 index 0000000000..104ab8aaee --- /dev/null +++ b/crates/parser/test_data/lexer/err/unclosed_raw_byte_string_with_unicode_escape.txt @@ -0,0 +1 @@ +BYTE_STRING "br##\"\\u{20AA}" error: Missing trailing `"` with `#` symbols to terminate the raw byte string literal diff --git a/crates/syntax/test_data/lexer/err/0033_unclosed_raw_string_at_eof.rs b/crates/parser/test_data/lexer/err/unclosed_raw_string_at_eof.rs index 557c59b625..557c59b625 100644 --- a/crates/syntax/test_data/lexer/err/0033_unclosed_raw_string_at_eof.rs +++ b/crates/parser/test_data/lexer/err/unclosed_raw_string_at_eof.rs diff --git a/crates/parser/test_data/lexer/err/unclosed_raw_string_at_eof.txt b/crates/parser/test_data/lexer/err/unclosed_raw_string_at_eof.txt new file mode 100644 index 0000000000..71b20fd19d --- /dev/null +++ b/crates/parser/test_data/lexer/err/unclosed_raw_string_at_eof.txt @@ -0,0 +1 @@ +STRING "r##\"" error: Missing trailing `"` with `#` symbols to terminate the raw string literal diff --git a/crates/syntax/test_data/lexer/err/0035_unclosed_raw_string_with_ascii_escape.rs b/crates/parser/test_data/lexer/err/unclosed_raw_string_with_ascii_escape.rs index 5bec883dc7..5bec883dc7 100644 --- a/crates/syntax/test_data/lexer/err/0035_unclosed_raw_string_with_ascii_escape.rs +++ b/crates/parser/test_data/lexer/err/unclosed_raw_string_with_ascii_escape.rs diff --git a/crates/parser/test_data/lexer/err/unclosed_raw_string_with_ascii_escape.txt b/crates/parser/test_data/lexer/err/unclosed_raw_string_with_ascii_escape.txt new file mode 100644 index 0000000000..dc106dd24a --- /dev/null +++ b/crates/parser/test_data/lexer/err/unclosed_raw_string_with_ascii_escape.txt @@ -0,0 +1 @@ +STRING "r##\"\\x7f" error: Missing trailing `"` with `#` symbols to terminate the raw string literal diff --git a/crates/syntax/test_data/lexer/err/0034_unclosed_raw_string_with_ferris.rs b/crates/parser/test_data/lexer/err/unclosed_raw_string_with_ferris.rs index bd046e4bb9..bd046e4bb9 100644 --- a/crates/syntax/test_data/lexer/err/0034_unclosed_raw_string_with_ferris.rs +++ b/crates/parser/test_data/lexer/err/unclosed_raw_string_with_ferris.rs diff --git a/crates/parser/test_data/lexer/err/unclosed_raw_string_with_ferris.txt b/crates/parser/test_data/lexer/err/unclosed_raw_string_with_ferris.txt new file mode 100644 index 0000000000..30ee029f65 --- /dev/null +++ b/crates/parser/test_data/lexer/err/unclosed_raw_string_with_ferris.txt @@ -0,0 +1 @@ +STRING "r##\"π¦" error: Missing trailing `"` with `#` symbols to terminate the raw string literal diff --git a/crates/syntax/test_data/lexer/err/0038_unclosed_raw_string_with_slash.rs b/crates/parser/test_data/lexer/err/unclosed_raw_string_with_slash.rs index 9242077b8b..9242077b8b 100644 --- a/crates/syntax/test_data/lexer/err/0038_unclosed_raw_string_with_slash.rs +++ b/crates/parser/test_data/lexer/err/unclosed_raw_string_with_slash.rs diff --git a/crates/parser/test_data/lexer/err/unclosed_raw_string_with_slash.txt b/crates/parser/test_data/lexer/err/unclosed_raw_string_with_slash.txt new file mode 100644 index 0000000000..8a6f6cc436 --- /dev/null +++ b/crates/parser/test_data/lexer/err/unclosed_raw_string_with_slash.txt @@ -0,0 +1 @@ +STRING "r##\"\\" error: Missing trailing `"` with `#` symbols to terminate the raw string literal diff --git a/crates/syntax/test_data/lexer/err/0039_unclosed_raw_string_with_slash_n.rs b/crates/parser/test_data/lexer/err/unclosed_raw_string_with_slash_n.rs index db1c16f2ba..db1c16f2ba 100644 --- a/crates/syntax/test_data/lexer/err/0039_unclosed_raw_string_with_slash_n.rs +++ b/crates/parser/test_data/lexer/err/unclosed_raw_string_with_slash_n.rs diff --git a/crates/parser/test_data/lexer/err/unclosed_raw_string_with_slash_n.txt b/crates/parser/test_data/lexer/err/unclosed_raw_string_with_slash_n.txt new file mode 100644 index 0000000000..f46eff2516 --- /dev/null +++ b/crates/parser/test_data/lexer/err/unclosed_raw_string_with_slash_n.txt @@ -0,0 +1 @@ +STRING "r##\"\\n" error: Missing trailing `"` with `#` symbols to terminate the raw string literal diff --git a/crates/syntax/test_data/lexer/err/0037_unclosed_raw_string_with_space.rs b/crates/parser/test_data/lexer/err/unclosed_raw_string_with_space.rs index f104bae4f2..f104bae4f2 100644 --- a/crates/syntax/test_data/lexer/err/0037_unclosed_raw_string_with_space.rs +++ b/crates/parser/test_data/lexer/err/unclosed_raw_string_with_space.rs diff --git a/crates/parser/test_data/lexer/err/unclosed_raw_string_with_space.txt b/crates/parser/test_data/lexer/err/unclosed_raw_string_with_space.txt new file mode 100644 index 0000000000..49b6afea45 --- /dev/null +++ b/crates/parser/test_data/lexer/err/unclosed_raw_string_with_space.txt @@ -0,0 +1 @@ +STRING "r##\" " error: Missing trailing `"` with `#` symbols to terminate the raw string literal diff --git a/crates/syntax/test_data/lexer/err/0036_unclosed_raw_string_with_unicode_escape.rs b/crates/parser/test_data/lexer/err/unclosed_raw_string_with_unicode_escape.rs index bf05c39134..bf05c39134 100644 --- a/crates/syntax/test_data/lexer/err/0036_unclosed_raw_string_with_unicode_escape.rs +++ b/crates/parser/test_data/lexer/err/unclosed_raw_string_with_unicode_escape.rs diff --git a/crates/parser/test_data/lexer/err/unclosed_raw_string_with_unicode_escape.txt b/crates/parser/test_data/lexer/err/unclosed_raw_string_with_unicode_escape.txt new file mode 100644 index 0000000000..d10d6d8e8c --- /dev/null +++ b/crates/parser/test_data/lexer/err/unclosed_raw_string_with_unicode_escape.txt @@ -0,0 +1 @@ +STRING "r##\"\\u{20AA}" error: Missing trailing `"` with `#` symbols to terminate the raw string literal diff --git a/crates/syntax/test_data/lexer/err/0017_unclosed_string_at_eof.rs b/crates/parser/test_data/lexer/err/unclosed_string_at_eof.rs index 9d68933c44..9d68933c44 100644 --- a/crates/syntax/test_data/lexer/err/0017_unclosed_string_at_eof.rs +++ b/crates/parser/test_data/lexer/err/unclosed_string_at_eof.rs diff --git a/crates/parser/test_data/lexer/err/unclosed_string_at_eof.txt b/crates/parser/test_data/lexer/err/unclosed_string_at_eof.txt new file mode 100644 index 0000000000..3b89ce0ca1 --- /dev/null +++ b/crates/parser/test_data/lexer/err/unclosed_string_at_eof.txt @@ -0,0 +1 @@ +STRING "\"" error: Missing trailing `"` symbol to terminate the string literal diff --git a/crates/syntax/test_data/lexer/err/0019_unclosed_string_with_ascii_escape.rs b/crates/parser/test_data/lexer/err/unclosed_string_with_ascii_escape.rs index 56186a3444..56186a3444 100644 --- a/crates/syntax/test_data/lexer/err/0019_unclosed_string_with_ascii_escape.rs +++ b/crates/parser/test_data/lexer/err/unclosed_string_with_ascii_escape.rs diff --git a/crates/parser/test_data/lexer/err/unclosed_string_with_ascii_escape.txt b/crates/parser/test_data/lexer/err/unclosed_string_with_ascii_escape.txt new file mode 100644 index 0000000000..6694cf17a6 --- /dev/null +++ b/crates/parser/test_data/lexer/err/unclosed_string_with_ascii_escape.txt @@ -0,0 +1 @@ +STRING "\"\\x7f" error: Missing trailing `"` symbol to terminate the string literal diff --git a/crates/syntax/test_data/lexer/err/0018_unclosed_string_with_ferris.rs b/crates/parser/test_data/lexer/err/unclosed_string_with_ferris.rs index d439b8d2a1..d439b8d2a1 100644 --- a/crates/syntax/test_data/lexer/err/0018_unclosed_string_with_ferris.rs +++ b/crates/parser/test_data/lexer/err/unclosed_string_with_ferris.rs diff --git a/crates/parser/test_data/lexer/err/unclosed_string_with_ferris.txt b/crates/parser/test_data/lexer/err/unclosed_string_with_ferris.txt new file mode 100644 index 0000000000..5f4501c18e --- /dev/null +++ b/crates/parser/test_data/lexer/err/unclosed_string_with_ferris.txt @@ -0,0 +1 @@ +STRING "\"π¦" error: Missing trailing `"` symbol to terminate the string literal diff --git a/crates/syntax/test_data/lexer/err/0022_unclosed_string_with_slash.rs b/crates/parser/test_data/lexer/err/unclosed_string_with_slash.rs index 00a2584008..00a2584008 100644 --- a/crates/syntax/test_data/lexer/err/0022_unclosed_string_with_slash.rs +++ b/crates/parser/test_data/lexer/err/unclosed_string_with_slash.rs diff --git a/crates/parser/test_data/lexer/err/unclosed_string_with_slash.txt b/crates/parser/test_data/lexer/err/unclosed_string_with_slash.txt new file mode 100644 index 0000000000..a8ac565ac8 --- /dev/null +++ b/crates/parser/test_data/lexer/err/unclosed_string_with_slash.txt @@ -0,0 +1 @@ +STRING "\"\\" error: Missing trailing `"` symbol to terminate the string literal diff --git a/crates/syntax/test_data/lexer/err/0024_unclosed_string_with_slash_double_quote.rs b/crates/parser/test_data/lexer/err/unclosed_string_with_slash_double_quote.rs index 403c2d6ddc..403c2d6ddc 100644 --- a/crates/syntax/test_data/lexer/err/0024_unclosed_string_with_slash_double_quote.rs +++ b/crates/parser/test_data/lexer/err/unclosed_string_with_slash_double_quote.rs diff --git a/crates/parser/test_data/lexer/err/unclosed_string_with_slash_double_quote.txt b/crates/parser/test_data/lexer/err/unclosed_string_with_slash_double_quote.txt new file mode 100644 index 0000000000..919183b919 --- /dev/null +++ b/crates/parser/test_data/lexer/err/unclosed_string_with_slash_double_quote.txt @@ -0,0 +1 @@ +STRING "\"\\\"" error: Missing trailing `"` symbol to terminate the string literal diff --git a/crates/syntax/test_data/lexer/err/0023_unclosed_string_with_slash_n.rs b/crates/parser/test_data/lexer/err/unclosed_string_with_slash_n.rs index a0c29b8cff..a0c29b8cff 100644 --- a/crates/syntax/test_data/lexer/err/0023_unclosed_string_with_slash_n.rs +++ b/crates/parser/test_data/lexer/err/unclosed_string_with_slash_n.rs diff --git a/crates/parser/test_data/lexer/err/unclosed_string_with_slash_n.txt b/crates/parser/test_data/lexer/err/unclosed_string_with_slash_n.txt new file mode 100644 index 0000000000..39e288af96 --- /dev/null +++ b/crates/parser/test_data/lexer/err/unclosed_string_with_slash_n.txt @@ -0,0 +1 @@ +STRING "\"\\n" error: Missing trailing `"` symbol to terminate the string literal diff --git a/crates/syntax/test_data/lexer/err/0021_unclosed_string_with_space.rs b/crates/parser/test_data/lexer/err/unclosed_string_with_space.rs index 72cdc841fb..72cdc841fb 100644 --- a/crates/syntax/test_data/lexer/err/0021_unclosed_string_with_space.rs +++ b/crates/parser/test_data/lexer/err/unclosed_string_with_space.rs diff --git a/crates/parser/test_data/lexer/err/unclosed_string_with_space.txt b/crates/parser/test_data/lexer/err/unclosed_string_with_space.txt new file mode 100644 index 0000000000..dcff94d7ed --- /dev/null +++ b/crates/parser/test_data/lexer/err/unclosed_string_with_space.txt @@ -0,0 +1 @@ +STRING "\" " error: Missing trailing `"` symbol to terminate the string literal diff --git a/crates/syntax/test_data/lexer/err/0020_unclosed_string_with_unicode_escape.rs b/crates/parser/test_data/lexer/err/unclosed_string_with_unicode_escape.rs index ed24095c3d..ed24095c3d 100644 --- a/crates/syntax/test_data/lexer/err/0020_unclosed_string_with_unicode_escape.rs +++ b/crates/parser/test_data/lexer/err/unclosed_string_with_unicode_escape.rs diff --git a/crates/parser/test_data/lexer/err/unclosed_string_with_unicode_escape.txt b/crates/parser/test_data/lexer/err/unclosed_string_with_unicode_escape.txt new file mode 100644 index 0000000000..ac232b530d --- /dev/null +++ b/crates/parser/test_data/lexer/err/unclosed_string_with_unicode_escape.txt @@ -0,0 +1 @@ +STRING "\"\\u{20AA}" error: Missing trailing `"` symbol to terminate the string literal diff --git a/crates/syntax/test_data/lexer/err/0048_unstarted_raw_byte_string_at_eof.rs b/crates/parser/test_data/lexer/err/unstarted_raw_byte_string_at_eof.rs index 7e8cadf4f4..7e8cadf4f4 100644 --- a/crates/syntax/test_data/lexer/err/0048_unstarted_raw_byte_string_at_eof.rs +++ b/crates/parser/test_data/lexer/err/unstarted_raw_byte_string_at_eof.rs diff --git a/crates/parser/test_data/lexer/err/unstarted_raw_byte_string_at_eof.txt b/crates/parser/test_data/lexer/err/unstarted_raw_byte_string_at_eof.txt new file mode 100644 index 0000000000..cf942c92f3 --- /dev/null +++ b/crates/parser/test_data/lexer/err/unstarted_raw_byte_string_at_eof.txt @@ -0,0 +1 @@ +BYTE_STRING "br##" error: Missing `"` symbol after `#` symbols to begin the raw byte string literal diff --git a/crates/syntax/test_data/lexer/err/0050_unstarted_raw_byte_string_with_ascii.rs b/crates/parser/test_data/lexer/err/unstarted_raw_byte_string_with_ascii.rs index d9b55455ac..d9b55455ac 100644 --- a/crates/syntax/test_data/lexer/err/0050_unstarted_raw_byte_string_with_ascii.rs +++ b/crates/parser/test_data/lexer/err/unstarted_raw_byte_string_with_ascii.rs diff --git a/crates/parser/test_data/lexer/err/unstarted_raw_byte_string_with_ascii.txt b/crates/parser/test_data/lexer/err/unstarted_raw_byte_string_with_ascii.txt new file mode 100644 index 0000000000..042769c275 --- /dev/null +++ b/crates/parser/test_data/lexer/err/unstarted_raw_byte_string_with_ascii.txt @@ -0,0 +1,9 @@ +BYTE_STRING "br## " error: Missing `"` symbol after `#` symbols to begin the raw byte string literal +IDENT "I" +WHITESPACE " " +IDENT "lack" +WHITESPACE " " +IDENT "a" +WHITESPACE " " +IDENT "quote" +BANG "!" diff --git a/crates/syntax/test_data/lexer/err/0047_unstarted_raw_string_at_eof.rs b/crates/parser/test_data/lexer/err/unstarted_raw_string_at_eof.rs index eddf8d0809..eddf8d0809 100644 --- a/crates/syntax/test_data/lexer/err/0047_unstarted_raw_string_at_eof.rs +++ b/crates/parser/test_data/lexer/err/unstarted_raw_string_at_eof.rs diff --git a/crates/parser/test_data/lexer/err/unstarted_raw_string_at_eof.txt b/crates/parser/test_data/lexer/err/unstarted_raw_string_at_eof.txt new file mode 100644 index 0000000000..2f7c7529a9 --- /dev/null +++ b/crates/parser/test_data/lexer/err/unstarted_raw_string_at_eof.txt @@ -0,0 +1 @@ +STRING "r##" error: Missing `"` symbol after `#` symbols to begin the raw string literal diff --git a/crates/syntax/test_data/lexer/err/0049_unstarted_raw_string_with_ascii.rs b/crates/parser/test_data/lexer/err/unstarted_raw_string_with_ascii.rs index 534668a9b6..534668a9b6 100644 --- a/crates/syntax/test_data/lexer/err/0049_unstarted_raw_string_with_ascii.rs +++ b/crates/parser/test_data/lexer/err/unstarted_raw_string_with_ascii.rs diff --git a/crates/parser/test_data/lexer/err/unstarted_raw_string_with_ascii.txt b/crates/parser/test_data/lexer/err/unstarted_raw_string_with_ascii.txt new file mode 100644 index 0000000000..4a06b0abe7 --- /dev/null +++ b/crates/parser/test_data/lexer/err/unstarted_raw_string_with_ascii.txt @@ -0,0 +1,9 @@ +STRING "r## " error: Missing `"` symbol after `#` symbols to begin the raw string literal +IDENT "I" +WHITESPACE " " +IDENT "lack" +WHITESPACE " " +IDENT "a" +WHITESPACE " " +IDENT "quote" +BANG "!" diff --git a/crates/syntax/test_data/lexer/ok/0012_block_comment.rs b/crates/parser/test_data/lexer/ok/block_comment.rs index b880a59d95..b880a59d95 100644 --- a/crates/syntax/test_data/lexer/ok/0012_block_comment.rs +++ b/crates/parser/test_data/lexer/ok/block_comment.rs diff --git a/crates/parser/test_data/lexer/ok/block_comment.txt b/crates/parser/test_data/lexer/ok/block_comment.txt new file mode 100644 index 0000000000..18bb5cad87 --- /dev/null +++ b/crates/parser/test_data/lexer/ok/block_comment.txt @@ -0,0 +1,6 @@ +COMMENT "/* */" +WHITESPACE "\n" +COMMENT "/**/" +WHITESPACE "\n" +COMMENT "/* /* */ */" +WHITESPACE "\n" diff --git a/crates/syntax/test_data/lexer/ok/0008_byte_strings.rs b/crates/parser/test_data/lexer/ok/byte_strings.rs index b54930f5e6..b54930f5e6 100644 --- a/crates/syntax/test_data/lexer/ok/0008_byte_strings.rs +++ b/crates/parser/test_data/lexer/ok/byte_strings.rs diff --git a/crates/parser/test_data/lexer/ok/byte_strings.txt b/crates/parser/test_data/lexer/ok/byte_strings.txt new file mode 100644 index 0000000000..c848ac368e --- /dev/null +++ b/crates/parser/test_data/lexer/ok/byte_strings.txt @@ -0,0 +1,22 @@ +BYTE "b''" +WHITESPACE " " +BYTE "b'x'" +WHITESPACE " " +BYTE_STRING "b\"foo\"" +WHITESPACE " " +BYTE_STRING "br\"\"" +WHITESPACE "\n" +BYTE "b''suf" +WHITESPACE " " +BYTE_STRING "b\"\"ix" +WHITESPACE " " +BYTE_STRING "br\"\"br" +WHITESPACE "\n" +BYTE "b'\\n'" +WHITESPACE " " +BYTE "b'\\\\'" +WHITESPACE " " +BYTE "b'\\''" +WHITESPACE " " +BYTE "b'hello'" +WHITESPACE "\n" diff --git a/crates/syntax/test_data/lexer/ok/0006_chars.rs b/crates/parser/test_data/lexer/ok/chars.rs index 454ee0a5f6..454ee0a5f6 100644 --- a/crates/syntax/test_data/lexer/ok/0006_chars.rs +++ b/crates/parser/test_data/lexer/ok/chars.rs diff --git a/crates/parser/test_data/lexer/ok/chars.txt b/crates/parser/test_data/lexer/ok/chars.txt new file mode 100644 index 0000000000..66e58cc298 --- /dev/null +++ b/crates/parser/test_data/lexer/ok/chars.txt @@ -0,0 +1,16 @@ +CHAR "'x'" +WHITESPACE " " +CHAR "' '" +WHITESPACE " " +CHAR "'0'" +WHITESPACE " " +CHAR "'hello'" +WHITESPACE " " +CHAR "'\\x7f'" +WHITESPACE " " +CHAR "'\\n'" +WHITESPACE " " +CHAR "'\\\\'" +WHITESPACE " " +CHAR "'\\''" +WHITESPACE "\n" diff --git a/crates/syntax/test_data/lexer/ok/0001_hello.rs b/crates/parser/test_data/lexer/ok/hello.rs index 95d09f2b10..95d09f2b10 100644 --- a/crates/syntax/test_data/lexer/ok/0001_hello.rs +++ b/crates/parser/test_data/lexer/ok/hello.rs diff --git a/crates/parser/test_data/lexer/ok/hello.txt b/crates/parser/test_data/lexer/ok/hello.txt new file mode 100644 index 0000000000..7f5ce9de1b --- /dev/null +++ b/crates/parser/test_data/lexer/ok/hello.txt @@ -0,0 +1,3 @@ +IDENT "hello" +WHITESPACE " " +IDENT "world" diff --git a/crates/syntax/test_data/lexer/ok/0003_ident.rs b/crates/parser/test_data/lexer/ok/ident.rs index c05c9c009d..c05c9c009d 100644 --- a/crates/syntax/test_data/lexer/ok/0003_ident.rs +++ b/crates/parser/test_data/lexer/ok/ident.rs diff --git a/crates/parser/test_data/lexer/ok/ident.txt b/crates/parser/test_data/lexer/ok/ident.txt new file mode 100644 index 0000000000..5689644c07 --- /dev/null +++ b/crates/parser/test_data/lexer/ok/ident.txt @@ -0,0 +1,14 @@ +IDENT "foo" +WHITESPACE " " +IDENT "foo_" +WHITESPACE " " +IDENT "_foo" +WHITESPACE " " +UNDERSCORE "_" +WHITESPACE " " +IDENT "__" +WHITESPACE " " +IDENT "x" +WHITESPACE " " +IDENT "ΠΏΡΠΈΠ²Π΅Ρ" +WHITESPACE "\n" diff --git a/crates/syntax/test_data/lexer/ok/0011_keywords.rs b/crates/parser/test_data/lexer/ok/keywords.rs index 1e91bff4e7..1e91bff4e7 100644 --- a/crates/syntax/test_data/lexer/ok/0011_keywords.rs +++ b/crates/parser/test_data/lexer/ok/keywords.rs diff --git a/crates/parser/test_data/lexer/ok/keywords.txt b/crates/parser/test_data/lexer/ok/keywords.txt new file mode 100644 index 0000000000..e19b1399aa --- /dev/null +++ b/crates/parser/test_data/lexer/ok/keywords.txt @@ -0,0 +1,64 @@ +ASYNC_KW "async" +WHITESPACE " " +FN_KW "fn" +WHITESPACE " " +USE_KW "use" +WHITESPACE " " +STRUCT_KW "struct" +WHITESPACE " " +TRAIT_KW "trait" +WHITESPACE " " +ENUM_KW "enum" +WHITESPACE " " +IMPL_KW "impl" +WHITESPACE " " +TRUE_KW "true" +WHITESPACE " " +FALSE_KW "false" +WHITESPACE " " +AS_KW "as" +WHITESPACE " " +EXTERN_KW "extern" +WHITESPACE " " +CRATE_KW "crate" +WHITESPACE "\n" +MOD_KW "mod" +WHITESPACE " " +PUB_KW "pub" +WHITESPACE " " +SELF_KW "self" +WHITESPACE " " +SUPER_KW "super" +WHITESPACE " " +IN_KW "in" +WHITESPACE " " +WHERE_KW "where" +WHITESPACE " " +FOR_KW "for" +WHITESPACE " " +LOOP_KW "loop" +WHITESPACE " " +WHILE_KW "while" +WHITESPACE " " +IF_KW "if" +WHITESPACE " " +MATCH_KW "match" +WHITESPACE " " +CONST_KW "const" +WHITESPACE "\n" +STATIC_KW "static" +WHITESPACE " " +MUT_KW "mut" +WHITESPACE " " +TYPE_KW "type" +WHITESPACE " " +REF_KW "ref" +WHITESPACE " " +LET_KW "let" +WHITESPACE " " +ELSE_KW "else" +WHITESPACE " " +MOVE_KW "move" +WHITESPACE " " +RETURN_KW "return" +WHITESPACE "\n" diff --git a/crates/syntax/test_data/lexer/ok/0007_lifetimes.rs b/crates/parser/test_data/lexer/ok/lifetimes.rs index b764f1dce3..b764f1dce3 100644 --- a/crates/syntax/test_data/lexer/ok/0007_lifetimes.rs +++ b/crates/parser/test_data/lexer/ok/lifetimes.rs diff --git a/crates/parser/test_data/lexer/ok/lifetimes.txt b/crates/parser/test_data/lexer/ok/lifetimes.txt new file mode 100644 index 0000000000..eeb1e95414 --- /dev/null +++ b/crates/parser/test_data/lexer/ok/lifetimes.txt @@ -0,0 +1,8 @@ +LIFETIME_IDENT "'a" +WHITESPACE " " +LIFETIME_IDENT "'foo" +WHITESPACE " " +LIFETIME_IDENT "'foo_bar_baz" +WHITESPACE " " +LIFETIME_IDENT "'_" +WHITESPACE "\n" diff --git a/crates/syntax/test_data/lexer/ok/0004_numbers.rs b/crates/parser/test_data/lexer/ok/numbers.rs index bc761c235d..bc761c235d 100644 --- a/crates/syntax/test_data/lexer/ok/0004_numbers.rs +++ b/crates/parser/test_data/lexer/ok/numbers.rs diff --git a/crates/parser/test_data/lexer/ok/numbers.txt b/crates/parser/test_data/lexer/ok/numbers.txt new file mode 100644 index 0000000000..8d13c3f610 --- /dev/null +++ b/crates/parser/test_data/lexer/ok/numbers.txt @@ -0,0 +1,57 @@ +INT_NUMBER "0" +WHITESPACE " " +INT_NUMBER "00" +WHITESPACE " " +INT_NUMBER "0_" +WHITESPACE " " +FLOAT_NUMBER "0." +WHITESPACE " " +INT_NUMBER "0z" +WHITESPACE "\n" +INT_NUMBER "01790" +WHITESPACE " " +INT_NUMBER "0b1790" +WHITESPACE " " +INT_NUMBER "0o1790" +WHITESPACE " " +INT_NUMBER "0x1790aAbBcCdDeEfF" +WHITESPACE " " +INT_NUMBER "001279" +WHITESPACE " " +INT_NUMBER "0_1279" +WHITESPACE " " +FLOAT_NUMBER "0.1279" +WHITESPACE " " +FLOAT_NUMBER "0e1279" +WHITESPACE " " +FLOAT_NUMBER "0E1279" +WHITESPACE "\n" +INT_NUMBER "0" +DOT "." +DOT "." +INT_NUMBER "2" +WHITESPACE "\n" +INT_NUMBER "0" +DOT "." +IDENT "foo" +L_PAREN "(" +R_PAREN ")" +WHITESPACE "\n" +FLOAT_NUMBER "0e+1" +WHITESPACE "\n" +INT_NUMBER "0" +DOT "." +IDENT "e" +PLUS "+" +INT_NUMBER "1" +WHITESPACE "\n" +FLOAT_NUMBER "0.0E-2" +WHITESPACE "\n" +FLOAT_NUMBER "0___0.10000____0000e+111__" +WHITESPACE "\n" +INT_NUMBER "1i64" +WHITESPACE " " +FLOAT_NUMBER "92.0f32" +WHITESPACE " " +INT_NUMBER "11__s" +WHITESPACE "\n" diff --git a/crates/syntax/test_data/lexer/ok/0014_raw_ident.rs b/crates/parser/test_data/lexer/ok/raw_ident.rs index b40a1b6a24..b40a1b6a24 100644 --- a/crates/syntax/test_data/lexer/ok/0014_raw_ident.rs +++ b/crates/parser/test_data/lexer/ok/raw_ident.rs diff --git a/crates/parser/test_data/lexer/ok/raw_ident.txt b/crates/parser/test_data/lexer/ok/raw_ident.txt new file mode 100644 index 0000000000..fddad99821 --- /dev/null +++ b/crates/parser/test_data/lexer/ok/raw_ident.txt @@ -0,0 +1,2 @@ +IDENT "r#raw_ident" +WHITESPACE "\n" diff --git a/crates/syntax/test_data/lexer/ok/0013_raw_strings.rs b/crates/parser/test_data/lexer/ok/raw_strings.rs index e5ed0b693b..e5ed0b693b 100644 --- a/crates/syntax/test_data/lexer/ok/0013_raw_strings.rs +++ b/crates/parser/test_data/lexer/ok/raw_strings.rs diff --git a/crates/parser/test_data/lexer/ok/raw_strings.txt b/crates/parser/test_data/lexer/ok/raw_strings.txt new file mode 100644 index 0000000000..13cf733b7d --- /dev/null +++ b/crates/parser/test_data/lexer/ok/raw_strings.txt @@ -0,0 +1,2 @@ +STRING "r###\"this is a r##\"raw\"## string\"###" +WHITESPACE "\n" diff --git a/crates/syntax/test_data/lexer/ok/0010_single_line_comments.rs b/crates/parser/test_data/lexer/ok/single_line_comments.rs index 4b6653f9cc..4b6653f9cc 100644 --- a/crates/syntax/test_data/lexer/ok/0010_single_line_comments.rs +++ b/crates/parser/test_data/lexer/ok/single_line_comments.rs diff --git a/crates/parser/test_data/lexer/ok/single_line_comments.txt b/crates/parser/test_data/lexer/ok/single_line_comments.txt new file mode 100644 index 0000000000..a7681e9f50 --- /dev/null +++ b/crates/parser/test_data/lexer/ok/single_line_comments.txt @@ -0,0 +1,22 @@ +SHEBANG "#!/usr/bin/env bash" +WHITESPACE "\n" +COMMENT "// hello" +WHITESPACE "\n" +COMMENT "//! World" +WHITESPACE "\n" +COMMENT "//!! Inner line doc" +WHITESPACE "\n" +COMMENT "/// Outer line doc" +WHITESPACE "\n" +COMMENT "//// Just a comment" +WHITESPACE "\n\n" +COMMENT "//" +WHITESPACE "\n" +COMMENT "//!" +WHITESPACE "\n" +COMMENT "//!!" +WHITESPACE "\n" +COMMENT "///" +WHITESPACE "\n" +COMMENT "////" +WHITESPACE "\n" diff --git a/crates/syntax/test_data/lexer/ok/0009_strings.rs b/crates/parser/test_data/lexer/ok/strings.rs index 4ddb5bffcc..4ddb5bffcc 100644 --- a/crates/syntax/test_data/lexer/ok/0009_strings.rs +++ b/crates/parser/test_data/lexer/ok/strings.rs diff --git a/crates/parser/test_data/lexer/ok/strings.txt b/crates/parser/test_data/lexer/ok/strings.txt new file mode 100644 index 0000000000..ec222591bd --- /dev/null +++ b/crates/parser/test_data/lexer/ok/strings.txt @@ -0,0 +1,8 @@ +STRING "\"hello\"" +WHITESPACE " " +STRING "r\"world\"" +WHITESPACE " " +STRING "\"\\n\\\"\\\\no escape\"" +WHITESPACE " " +STRING "\"multi\nline\"" +WHITESPACE "\n" diff --git a/crates/syntax/test_data/lexer/ok/0005_symbols.rs b/crates/parser/test_data/lexer/ok/symbols.rs index 487569b5ae..487569b5ae 100644 --- a/crates/syntax/test_data/lexer/ok/0005_symbols.rs +++ b/crates/parser/test_data/lexer/ok/symbols.rs diff --git a/crates/parser/test_data/lexer/ok/symbols.txt b/crates/parser/test_data/lexer/ok/symbols.txt new file mode 100644 index 0000000000..533ccff9a8 --- /dev/null +++ b/crates/parser/test_data/lexer/ok/symbols.txt @@ -0,0 +1,77 @@ +SEMICOLON ";" +WHITESPACE " " +COMMA "," +WHITESPACE " " +L_PAREN "(" +WHITESPACE " " +R_PAREN ")" +WHITESPACE " " +L_CURLY "{" +WHITESPACE " " +R_CURLY "}" +WHITESPACE " " +L_BRACK "[" +WHITESPACE " " +R_BRACK "]" +WHITESPACE " " +L_ANGLE "<" +WHITESPACE " " +R_ANGLE ">" +WHITESPACE " " +AT "@" +WHITESPACE " " +POUND "#" +WHITESPACE " " +TILDE "~" +WHITESPACE " " +QUESTION "?" +WHITESPACE " " +DOLLAR "$" +WHITESPACE " " +AMP "&" +WHITESPACE " " +PIPE "|" +WHITESPACE " " +PLUS "+" +WHITESPACE " " +STAR "*" +WHITESPACE " " +SLASH "/" +WHITESPACE " " +CARET "^" +WHITESPACE " " +PERCENT "%" +WHITESPACE "\n" +DOT "." +WHITESPACE " " +DOT "." +DOT "." +WHITESPACE " " +DOT "." +DOT "." +DOT "." +WHITESPACE " " +DOT "." +DOT "." +EQ "=" +WHITESPACE "\n" +COLON ":" +WHITESPACE " " +COLON ":" +COLON ":" +WHITESPACE "\n" +EQ "=" +WHITESPACE " " +EQ "=" +R_ANGLE ">" +WHITESPACE "\n" +BANG "!" +WHITESPACE " " +BANG "!" +EQ "=" +WHITESPACE "\n" +MINUS "-" +WHITESPACE " " +MINUS "-" +R_ANGLE ">" +WHITESPACE "\n" diff --git a/crates/syntax/test_data/lexer/ok/0002_whitespace.rs b/crates/parser/test_data/lexer/ok/whitespace.rs index 08fce1418a..08fce1418a 100644 --- a/crates/syntax/test_data/lexer/ok/0002_whitespace.rs +++ b/crates/parser/test_data/lexer/ok/whitespace.rs diff --git a/crates/parser/test_data/lexer/ok/whitespace.txt b/crates/parser/test_data/lexer/ok/whitespace.txt new file mode 100644 index 0000000000..8ccb79e4ec --- /dev/null +++ b/crates/parser/test_data/lexer/ok/whitespace.txt @@ -0,0 +1,12 @@ +IDENT "a" +WHITESPACE " " +IDENT "b" +WHITESPACE " " +IDENT "c" +WHITESPACE "\n" +IDENT "d" +WHITESPACE "\n\n" +IDENT "e" +WHITESPACE "\t" +IDENT "f" +WHITESPACE "\n" diff --git a/crates/syntax/src/lib.rs b/crates/syntax/src/lib.rs index ec281625a6..65a6b7ac4e 100644 --- a/crates/syntax/src/lib.rs +++ b/crates/syntax/src/lib.rs @@ -48,7 +48,6 @@ use text_edit::Indel; pub use crate::{ ast::{AstNode, AstToken}, - parsing::lexer::{lex_single_syntax_kind, lex_single_valid_syntax_kind, tokenize, Token}, ptr::{AstPtr, SyntaxNodePtr}, syntax_error::SyntaxError, syntax_node::{ diff --git a/crates/syntax/src/parsing.rs b/crates/syntax/src/parsing.rs index 865e146482..cba1ddde85 100644 --- a/crates/syntax/src/parsing.rs +++ b/crates/syntax/src/parsing.rs @@ -1,7 +1,6 @@ //! Lexing, bridging to parser (which does the actual parsing) and //! incremental reparsing. -pub(crate) mod lexer; mod text_tree_sink; mod reparsing; @@ -10,18 +9,17 @@ use text_tree_sink::TextTreeSink; use crate::{syntax_node::GreenNode, AstNode, SyntaxError, SyntaxNode}; -pub(crate) use crate::parsing::{lexer::*, reparsing::incremental_reparse}; +pub(crate) use crate::parsing::reparsing::incremental_reparse; pub(crate) fn parse_text(text: &str) -> (GreenNode, Vec<SyntaxError>) { - let (lexer_tokens, lexer_errors) = tokenize(text); - let parser_tokens = to_parser_tokens(text, &lexer_tokens); + let lexed = parser::LexedStr::new(text); + let parser_tokens = lexed.to_tokens(); - let mut tree_sink = TextTreeSink::new(text, &lexer_tokens); + let mut tree_sink = TextTreeSink::new(lexed); parser::parse_source_file(&parser_tokens, &mut tree_sink); - let (tree, mut parser_errors) = tree_sink.finish(); - parser_errors.extend(lexer_errors); + let (tree, parser_errors) = tree_sink.finish(); (tree, parser_errors) } @@ -31,14 +29,13 @@ pub(crate) fn parse_text_as<T: AstNode>( text: &str, entry_point: parser::ParserEntryPoint, ) -> Result<T, ()> { - let (lexer_tokens, lexer_errors) = tokenize(text); - if !lexer_errors.is_empty() { + let lexed = parser::LexedStr::new(text); + if lexed.errors().next().is_some() { return Err(()); } + let parser_tokens = lexed.to_tokens(); - let parser_tokens = to_parser_tokens(text, &lexer_tokens); - - let mut tree_sink = TextTreeSink::new(text, &lexer_tokens); + let mut tree_sink = TextTreeSink::new(lexed); // TextTreeSink assumes that there's at least some root node to which it can attach errors and // tokens. We arbitrarily give it a SourceFile. @@ -54,29 +51,3 @@ pub(crate) fn parse_text_as<T: AstNode>( SyntaxNode::new_root(tree).first_child().and_then(T::cast).ok_or(()) } - -pub(crate) fn to_parser_tokens(text: &str, lexer_tokens: &[lexer::Token]) -> ::parser::Tokens { - let mut off = 0; - let mut res = parser::Tokens::default(); - let mut was_joint = false; - for t in lexer_tokens { - if t.kind.is_trivia() { - was_joint = false; - } else { - if t.kind == SyntaxKind::IDENT { - let token_text = &text[off..][..usize::from(t.len)]; - let contextual_kw = - SyntaxKind::from_contextual_keyword(token_text).unwrap_or(SyntaxKind::IDENT); - res.push_ident(contextual_kw); - } else { - if was_joint { - res.was_joint(); - } - res.push(t.kind); - } - was_joint = true; - } - off += usize::from(t.len); - } - res -} diff --git a/crates/syntax/src/parsing/lexer.rs b/crates/syntax/src/parsing/lexer.rs deleted file mode 100644 index ae4844e486..0000000000 --- a/crates/syntax/src/parsing/lexer.rs +++ /dev/null @@ -1,261 +0,0 @@ -//! Lexer analyzes raw input string and produces lexemes (tokens). -//! It is just a bridge to `rustc_lexer`. - -use std::convert::TryInto; - -use rustc_lexer::RawStrError; - -use crate::{ - SyntaxError, - SyntaxKind::{self, *}, - TextRange, TextSize, T, -}; - -/// A token of Rust source. -#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] -pub struct Token { - /// The kind of token. - pub kind: SyntaxKind, - /// The length of the token. - pub len: TextSize, -} - -/// Break a string up into its component tokens. -/// Beware that it checks for shebang first and its length contributes to resulting -/// tokens offsets. -pub fn tokenize(text: &str) -> (Vec<Token>, Vec<SyntaxError>) { - // non-empty string is a precondition of `rustc_lexer::strip_shebang()`. - if text.is_empty() { - return Default::default(); - } - - let mut tokens = Vec::new(); - let mut errors = Vec::new(); - - let mut offset = match rustc_lexer::strip_shebang(text) { - Some(shebang_len) => { - tokens.push(Token { kind: SHEBANG, len: shebang_len.try_into().unwrap() }); - shebang_len - } - None => 0, - }; - - let text_without_shebang = &text[offset..]; - - for rustc_token in rustc_lexer::tokenize(text_without_shebang) { - let token_len: TextSize = rustc_token.len.try_into().unwrap(); - let token_range = TextRange::at(offset.try_into().unwrap(), token_len); - - let (syntax_kind, err_message) = - rustc_token_kind_to_syntax_kind(&rustc_token.kind, &text[token_range]); - - tokens.push(Token { kind: syntax_kind, len: token_len }); - - if let Some(err_message) = err_message { - errors.push(SyntaxError::new(err_message, token_range)); - } - - offset += rustc_token.len; - } - - (tokens, errors) -} - -/// Returns `SyntaxKind` and `Option<SyntaxError>` if `text` parses as a single token. -/// -/// Returns `None` if the string contains zero *or two or more* tokens. -/// The token is malformed if the returned error is not `None`. -/// -/// Beware that unescape errors are not checked at tokenization time. -pub fn lex_single_syntax_kind(text: &str) -> Option<(SyntaxKind, Option<SyntaxError>)> { - let (first_token, err) = lex_first_token(text)?; - if first_token.len != TextSize::of(text) { - return None; - } - Some((first_token.kind, err)) -} - -/// The same as `lex_single_syntax_kind()` but returns only `SyntaxKind` and -/// returns `None` if any tokenization error occurred. -/// -/// Beware that unescape errors are not checked at tokenization time. -pub fn lex_single_valid_syntax_kind(text: &str) -> Option<SyntaxKind> { - let (single_token, err) = lex_single_syntax_kind(text)?; - if err.is_some() { - return None; - } - Some(single_token) -} - -/// Returns `SyntaxKind` and `Option<SyntaxError>` of the first token -/// encountered at the beginning of the string. -/// -/// Returns `None` if the string contains zero tokens or if the token was parsed -/// with an error. -/// The token is malformed if the returned error is not `None`. -/// -/// Beware that unescape errors are not checked at tokenization time. -fn lex_first_token(text: &str) -> Option<(Token, Option<SyntaxError>)> { - // non-empty string is a precondition of `rustc_lexer::first_token()`. - if text.is_empty() { - return None; - } - - let rustc_token = rustc_lexer::first_token(text); - let (syntax_kind, err_message) = rustc_token_kind_to_syntax_kind(&rustc_token.kind, text); - - let token = Token { kind: syntax_kind, len: rustc_token.len.try_into().unwrap() }; - let optional_error = err_message - .map(|err_message| SyntaxError::new(err_message, TextRange::up_to(TextSize::of(text)))); - - Some((token, optional_error)) -} - -/// Returns `SyntaxKind` and an optional tokenize error message. -fn rustc_token_kind_to_syntax_kind( - rustc_token_kind: &rustc_lexer::TokenKind, - token_text: &str, -) -> (SyntaxKind, Option<&'static str>) { - // A note on an intended tradeoff: - // We drop some useful information here (see patterns with double dots `..`) - // Storing that info in `SyntaxKind` is not possible due to its layout requirements of - // being `u16` that come from `rowan::SyntaxKind`. - - let syntax_kind = { - match rustc_token_kind { - rustc_lexer::TokenKind::LineComment { doc_style: _ } => COMMENT, - - rustc_lexer::TokenKind::BlockComment { doc_style: _, terminated: true } => COMMENT, - rustc_lexer::TokenKind::BlockComment { doc_style: _, terminated: false } => { - return ( - COMMENT, - Some("Missing trailing `*/` symbols to terminate the block comment"), - ); - } - - rustc_lexer::TokenKind::Whitespace => WHITESPACE, - - rustc_lexer::TokenKind::Ident => { - if token_text == "_" { - UNDERSCORE - } else { - SyntaxKind::from_keyword(token_text).unwrap_or(IDENT) - } - } - - rustc_lexer::TokenKind::RawIdent => IDENT, - rustc_lexer::TokenKind::Literal { kind, .. } => return match_literal_kind(kind), - - rustc_lexer::TokenKind::Lifetime { starts_with_number: false } => LIFETIME_IDENT, - rustc_lexer::TokenKind::Lifetime { starts_with_number: true } => { - return (LIFETIME_IDENT, Some("Lifetime name cannot start with a number")) - } - - rustc_lexer::TokenKind::Semi => T![;], - rustc_lexer::TokenKind::Comma => T![,], - rustc_lexer::TokenKind::Dot => T![.], - rustc_lexer::TokenKind::OpenParen => T!['('], - rustc_lexer::TokenKind::CloseParen => T![')'], - rustc_lexer::TokenKind::OpenBrace => T!['{'], - rustc_lexer::TokenKind::CloseBrace => T!['}'], - rustc_lexer::TokenKind::OpenBracket => T!['['], - rustc_lexer::TokenKind::CloseBracket => T![']'], - rustc_lexer::TokenKind::At => T![@], - rustc_lexer::TokenKind::Pound => T![#], - rustc_lexer::TokenKind::Tilde => T![~], - rustc_lexer::TokenKind::Question => T![?], - rustc_lexer::TokenKind::Colon => T![:], - rustc_lexer::TokenKind::Dollar => T![$], - rustc_lexer::TokenKind::Eq => T![=], - rustc_lexer::TokenKind::Bang => T![!], - rustc_lexer::TokenKind::Lt => T![<], - rustc_lexer::TokenKind::Gt => T![>], - rustc_lexer::TokenKind::Minus => T![-], - rustc_lexer::TokenKind::And => T![&], - rustc_lexer::TokenKind::Or => T![|], - rustc_lexer::TokenKind::Plus => T![+], - rustc_lexer::TokenKind::Star => T![*], - rustc_lexer::TokenKind::Slash => T![/], - rustc_lexer::TokenKind::Caret => T![^], - rustc_lexer::TokenKind::Percent => T![%], - rustc_lexer::TokenKind::Unknown => ERROR, - } - }; - - return (syntax_kind, None); - - fn match_literal_kind(kind: &rustc_lexer::LiteralKind) -> (SyntaxKind, Option<&'static str>) { - let mut err = ""; - let syntax_kind = match *kind { - rustc_lexer::LiteralKind::Int { empty_int, base: _ } => { - if empty_int { - err = "Missing digits after the integer base prefix"; - } - INT_NUMBER - } - rustc_lexer::LiteralKind::Float { empty_exponent, base: _ } => { - if empty_exponent { - err = "Missing digits after the exponent symbol"; - } - FLOAT_NUMBER - } - rustc_lexer::LiteralKind::Char { terminated } => { - if !terminated { - err = "Missing trailing `'` symbol to terminate the character literal"; - } - CHAR - } - rustc_lexer::LiteralKind::Byte { terminated } => { - if !terminated { - err = "Missing trailing `'` symbol to terminate the byte literal"; - } - BYTE - } - rustc_lexer::LiteralKind::Str { terminated } => { - if !terminated { - err = "Missing trailing `\"` symbol to terminate the string literal"; - } - STRING - } - rustc_lexer::LiteralKind::ByteStr { terminated } => { - if !terminated { - err = "Missing trailing `\"` symbol to terminate the byte string literal"; - } - BYTE_STRING - } - rustc_lexer::LiteralKind::RawStr { err: raw_str_err, .. } => { - if let Some(raw_str_err) = raw_str_err { - err = match raw_str_err { - RawStrError::InvalidStarter { .. } => "Missing `\"` symbol after `#` symbols to begin the raw string literal", - RawStrError::NoTerminator { expected, found, .. } => if expected == found { - "Missing trailing `\"` to terminate the raw string literal" - } else { - "Missing trailing `\"` with `#` symbols to terminate the raw string literal" - }, - RawStrError::TooManyDelimiters { .. } => "Too many `#` symbols: raw strings may be delimited by up to 65535 `#` symbols", - }; - }; - STRING - } - rustc_lexer::LiteralKind::RawByteStr { err: raw_str_err, .. } => { - if let Some(raw_str_err) = raw_str_err { - err = match raw_str_err { - RawStrError::InvalidStarter { .. } => "Missing `\"` symbol after `#` symbols to begin the raw byte string literal", - RawStrError::NoTerminator { expected, found, .. } => if expected == found { - "Missing trailing `\"` to terminate the raw byte string literal" - } else { - "Missing trailing `\"` with `#` symbols to terminate the raw byte string literal" - }, - RawStrError::TooManyDelimiters { .. } => "Too many `#` symbols: raw byte strings may be delimited by up to 65535 `#` symbols", - }; - }; - - BYTE_STRING - } - }; - - let err = if err.is_empty() { None } else { Some(err) }; - - (syntax_kind, err) - } -} diff --git a/crates/syntax/src/parsing/reparsing.rs b/crates/syntax/src/parsing/reparsing.rs index 62f39a9347..e9567a838c 100644 --- a/crates/syntax/src/parsing/reparsing.rs +++ b/crates/syntax/src/parsing/reparsing.rs @@ -10,11 +10,7 @@ use parser::Reparser; use text_edit::Indel; use crate::{ - parsing::{ - lexer::{lex_single_syntax_kind, tokenize, Token}, - text_tree_sink::TextTreeSink, - to_parser_tokens, - }, + parsing::text_tree_sink::TextTreeSink, syntax_node::{GreenNode, GreenToken, NodeOrToken, SyntaxElement, SyntaxNode}, SyntaxError, SyntaxKind::*, @@ -53,7 +49,7 @@ fn reparse_token( } let mut new_text = get_text_after_edit(prev_token.clone().into(), edit); - let (new_token_kind, new_err) = lex_single_syntax_kind(&new_text)?; + let (new_token_kind, new_err) = parser::LexedStr::single_token(&new_text)?; if new_token_kind != prev_token_kind || (new_token_kind == IDENT && is_contextual_kw(&new_text)) @@ -66,7 +62,7 @@ fn reparse_token( // `b` no longer remains an identifier, but becomes a part of byte string literal if let Some(next_char) = root.text().char_at(prev_token.text_range().end()) { new_text.push(next_char); - let token_with_next_char = lex_single_syntax_kind(&new_text); + let token_with_next_char = parser::LexedStr::single_token(&new_text); if let Some((_kind, _error)) = token_with_next_char { return None; } @@ -74,9 +70,10 @@ fn reparse_token( } let new_token = GreenToken::new(rowan::SyntaxKind(prev_token_kind.into()), &new_text); + let range = TextRange::up_to(TextSize::of(&new_text)); Some(( prev_token.replace_with(new_token), - new_err.into_iter().collect(), + new_err.into_iter().map(|msg| SyntaxError::new(msg, range)).collect(), prev_token.text_range(), )) } @@ -91,17 +88,17 @@ fn reparse_block( let (node, reparser) = find_reparsable_node(root, edit.delete)?; let text = get_text_after_edit(node.clone().into(), edit); - let (lexer_tokens, new_lexer_errors) = tokenize(&text); - if !is_balanced(&lexer_tokens) { + let lexed = parser::LexedStr::new(text.as_str()); + let parser_tokens = lexed.to_tokens(); + if !is_balanced(&lexed) { return None; } - let parser_tokens = to_parser_tokens(&text, &lexer_tokens); - let mut tree_sink = TextTreeSink::new(&text, &lexer_tokens); + let mut tree_sink = TextTreeSink::new(lexed); + reparser.parse(&parser_tokens, &mut tree_sink); - let (green, mut new_parser_errors) = tree_sink.finish(); - new_parser_errors.extend(new_lexer_errors); + let (green, new_parser_errors) = tree_sink.finish(); Some((node.replace_with(green), new_parser_errors, node.text_range())) } @@ -131,16 +128,13 @@ fn find_reparsable_node(node: &SyntaxNode, range: TextRange) -> Option<(SyntaxNo }) } -fn is_balanced(tokens: &[Token]) -> bool { - if tokens.is_empty() - || tokens.first().unwrap().kind != T!['{'] - || tokens.last().unwrap().kind != T!['}'] - { +fn is_balanced(lexed: &parser::LexedStr<'_>) -> bool { + if lexed.is_empty() || lexed.kind(0) != T!['{'] || lexed.kind(lexed.len() - 1) != T!['}'] { return false; } let mut balance = 0usize; - for t in &tokens[1..tokens.len() - 1] { - match t.kind { + for i in 1..lexed.len() - 1 { + match lexed.kind(i) { T!['{'] => balance += 1, T!['}'] => { balance = match balance.checked_sub(1) { diff --git a/crates/syntax/src/parsing/text_tree_sink.rs b/crates/syntax/src/parsing/text_tree_sink.rs index c1792199fd..c9e7feb965 100644 --- a/crates/syntax/src/parsing/text_tree_sink.rs +++ b/crates/syntax/src/parsing/text_tree_sink.rs @@ -2,25 +2,22 @@ use std::mem; -use parser::{ParseError, TreeSink}; +use parser::{LexedStr, ParseError, TreeSink}; use crate::{ ast, - parsing::Token, syntax_node::GreenNode, SyntaxError, SyntaxKind::{self, *}, - SyntaxTreeBuilder, TextRange, TextSize, + SyntaxTreeBuilder, TextRange, }; /// Bridges the parser with our specific syntax tree representation. /// /// `TextTreeSink` also handles attachment of trivia (whitespace) to nodes. pub(crate) struct TextTreeSink<'a> { - text: &'a str, - tokens: &'a [Token], - text_pos: TextSize, - token_pos: usize, + lexed: LexedStr<'a>, + pos: usize, state: State, inner: SyntaxTreeBuilder, } @@ -39,12 +36,7 @@ impl<'a> TreeSink for TextTreeSink<'a> { State::Normal => (), } self.eat_trivias(); - let n_tokens = n_tokens as usize; - let len = self.tokens[self.token_pos..self.token_pos + n_tokens] - .iter() - .map(|it| it.len) - .sum::<TextSize>(); - self.do_token(kind, len, n_tokens); + self.do_token(kind, n_tokens as usize); } fn start_node(&mut self, kind: SyntaxKind) { @@ -60,20 +52,12 @@ impl<'a> TreeSink for TextTreeSink<'a> { } let n_trivias = - self.tokens[self.token_pos..].iter().take_while(|it| it.kind.is_trivia()).count(); - let leading_trivias = &self.tokens[self.token_pos..self.token_pos + n_trivias]; - let mut trivia_end = - self.text_pos + leading_trivias.iter().map(|it| it.len).sum::<TextSize>(); - - let n_attached_trivias = { - let leading_trivias = leading_trivias.iter().rev().map(|it| { - let next_end = trivia_end - it.len; - let range = TextRange::new(next_end, trivia_end); - trivia_end = next_end; - (it.kind, &self.text[range]) - }); - n_attached_trivias(kind, leading_trivias) - }; + (self.pos..self.lexed.len()).take_while(|&it| self.lexed.kind(it).is_trivia()).count(); + let leading_trivias = self.pos..self.pos + n_trivias; + let n_attached_trivias = n_attached_trivias( + kind, + leading_trivias.rev().map(|it| (self.lexed.kind(it), self.lexed.text(it))), + ); self.eat_n_trivias(n_trivias - n_attached_trivias); self.inner.start_node(kind); self.eat_n_trivias(n_attached_trivias); @@ -88,20 +72,14 @@ impl<'a> TreeSink for TextTreeSink<'a> { } fn error(&mut self, error: ParseError) { - self.inner.error(error, self.text_pos); + let text_pos = self.lexed.text_start(self.pos).try_into().unwrap(); + self.inner.error(error, text_pos); } } impl<'a> TextTreeSink<'a> { - pub(super) fn new(text: &'a str, tokens: &'a [Token]) -> Self { - Self { - text, - tokens, - text_pos: 0.into(), - token_pos: 0, - state: State::PendingStart, - inner: SyntaxTreeBuilder::default(), - } + pub(super) fn new(lexed: parser::LexedStr<'a>) -> Self { + Self { lexed, pos: 0, state: State::PendingStart, inner: SyntaxTreeBuilder::default() } } pub(super) fn finish_eof(mut self) -> (GreenNode, Vec<SyntaxError>, bool) { @@ -113,8 +91,17 @@ impl<'a> TextTreeSink<'a> { State::PendingStart | State::Normal => unreachable!(), } - let (node, errors) = self.inner.finish_raw(); - let is_eof = self.token_pos == self.tokens.len(); + let (node, mut errors) = self.inner.finish_raw(); + for (i, err) in self.lexed.errors() { + let text_range = self.lexed.text_range(i); + let text_range = TextRange::new( + text_range.start.try_into().unwrap(), + text_range.end.try_into().unwrap(), + ); + errors.push(SyntaxError::new(err, text_range)) + } + + let is_eof = self.pos == self.lexed.len(); (node, errors, is_eof) } @@ -125,27 +112,26 @@ impl<'a> TextTreeSink<'a> { } fn eat_trivias(&mut self) { - while let Some(&token) = self.tokens.get(self.token_pos) { - if !token.kind.is_trivia() { + while self.pos < self.lexed.len() { + let kind = self.lexed.kind(self.pos); + if !kind.is_trivia() { break; } - self.do_token(token.kind, token.len, 1); + self.do_token(kind, 1); } } fn eat_n_trivias(&mut self, n: usize) { for _ in 0..n { - let token = self.tokens[self.token_pos]; - assert!(token.kind.is_trivia()); - self.do_token(token.kind, token.len, 1); + let kind = self.lexed.kind(self.pos); + assert!(kind.is_trivia()); + self.do_token(kind, 1); } } - fn do_token(&mut self, kind: SyntaxKind, len: TextSize, n_tokens: usize) { - let range = TextRange::at(self.text_pos, len); - let text = &self.text[range]; - self.text_pos += len; - self.token_pos += n_tokens; + fn do_token(&mut self, kind: SyntaxKind, n_tokens: usize) { + let text = &self.lexed.range_text(self.pos..self.pos + n_tokens); + self.pos += n_tokens; self.inner.token(kind, text); } } diff --git a/crates/syntax/src/tests.rs b/crates/syntax/src/tests.rs index 022db39f33..69c5b1cd35 100644 --- a/crates/syntax/src/tests.rs +++ b/crates/syntax/src/tests.rs @@ -3,7 +3,6 @@ mod sourcegen_ast; mod ast_src; use std::{ - fmt::Write, fs, path::{Path, PathBuf}, }; @@ -13,25 +12,7 @@ use expect_test::expect_file; use rayon::prelude::*; use test_utils::{bench, bench_fixture, project_root}; -use crate::{ast, fuzz, tokenize, AstNode, SourceFile, SyntaxError, TextRange, TextSize, Token}; - -#[test] -fn lexer_tests() { - // FIXME: - // * Add tests for unicode escapes in byte-character and [raw]-byte-string literals - // * Add tests for unescape errors - - dir_tests(&test_data_dir(), &["lexer/ok"], "txt", |text, path| { - let (tokens, errors) = tokenize(text); - assert_errors_are_absent(&errors, path); - dump_tokens_and_errors(&tokens, &errors, text) - }); - dir_tests(&test_data_dir(), &["lexer/err"], "txt", |text, path| { - let (tokens, errors) = tokenize(text); - assert_errors_are_present(&errors, path); - dump_tokens_and_errors(&tokens, &errors, text) - }); -} +use crate::{ast, fuzz, AstNode, SourceFile, SyntaxError}; #[test] fn parse_smoke_test() { @@ -206,22 +187,6 @@ fn assert_errors_are_absent(errors: &[SyntaxError], path: &Path) { ); } -fn dump_tokens_and_errors(tokens: &[Token], errors: &[SyntaxError], text: &str) -> String { - let mut acc = String::new(); - let mut offset: TextSize = 0.into(); - for token in tokens { - let token_len = token.len; - let token_text = &text[TextRange::at(offset, token.len)]; - offset += token.len; - writeln!(acc, "{:?} {:?} {:?}", token.kind, token_len, token_text).unwrap(); - } - for err in errors { - writeln!(acc, "> error{:?} token({:?}) msg({})", err.range(), &text[err.range()], err) - .unwrap(); - } - acc -} - fn fragment_parser_dir_test<T, F>(ok_paths: &[&str], err_paths: &[&str], f: F) where T: crate::AstNode, diff --git a/crates/syntax/test_data/lexer/err/0001_unclosed_char_at_eof.txt b/crates/syntax/test_data/lexer/err/0001_unclosed_char_at_eof.txt deleted file mode 100644 index 135f49552b..0000000000 --- a/crates/syntax/test_data/lexer/err/0001_unclosed_char_at_eof.txt +++ /dev/null @@ -1,2 +0,0 @@ -CHAR 1 "'" -> error0..1 token("'") msg(Missing trailing `'` symbol to terminate the character literal) diff --git a/crates/syntax/test_data/lexer/err/0002_unclosed_char_with_ferris.txt b/crates/syntax/test_data/lexer/err/0002_unclosed_char_with_ferris.txt deleted file mode 100644 index cc3933d956..0000000000 --- a/crates/syntax/test_data/lexer/err/0002_unclosed_char_with_ferris.txt +++ /dev/null @@ -1,2 +0,0 @@ -CHAR 5 "'π¦" -> error0..5 token("'π¦") msg(Missing trailing `'` symbol to terminate the character literal) diff --git a/crates/syntax/test_data/lexer/err/0003_unclosed_char_with_ascii_escape.txt b/crates/syntax/test_data/lexer/err/0003_unclosed_char_with_ascii_escape.txt deleted file mode 100644 index 21d990e6f1..0000000000 --- a/crates/syntax/test_data/lexer/err/0003_unclosed_char_with_ascii_escape.txt +++ /dev/null @@ -1,2 +0,0 @@ -CHAR 5 "'\\x7f" -> error0..5 token("'\\x7f") msg(Missing trailing `'` symbol to terminate the character literal) diff --git a/crates/syntax/test_data/lexer/err/0004_unclosed_char_with_unicode_escape.txt b/crates/syntax/test_data/lexer/err/0004_unclosed_char_with_unicode_escape.txt deleted file mode 100644 index 055dba64cc..0000000000 --- a/crates/syntax/test_data/lexer/err/0004_unclosed_char_with_unicode_escape.txt +++ /dev/null @@ -1,2 +0,0 @@ -CHAR 9 "'\\u{20AA}" -> error0..9 token("'\\u{20AA}") msg(Missing trailing `'` symbol to terminate the character literal) diff --git a/crates/syntax/test_data/lexer/err/0005_unclosed_char_with_space.txt b/crates/syntax/test_data/lexer/err/0005_unclosed_char_with_space.txt deleted file mode 100644 index 9ee5e93fa4..0000000000 --- a/crates/syntax/test_data/lexer/err/0005_unclosed_char_with_space.txt +++ /dev/null @@ -1,2 +0,0 @@ -CHAR 2 "' " -> error0..2 token("' ") msg(Missing trailing `'` symbol to terminate the character literal) diff --git a/crates/syntax/test_data/lexer/err/0006_unclosed_char_with_slash.txt b/crates/syntax/test_data/lexer/err/0006_unclosed_char_with_slash.txt deleted file mode 100644 index dc3a596f63..0000000000 --- a/crates/syntax/test_data/lexer/err/0006_unclosed_char_with_slash.txt +++ /dev/null @@ -1,2 +0,0 @@ -CHAR 2 "'\\" -> error0..2 token("'\\") msg(Missing trailing `'` symbol to terminate the character literal) diff --git a/crates/syntax/test_data/lexer/err/0007_unclosed_char_with_slash_n.txt b/crates/syntax/test_data/lexer/err/0007_unclosed_char_with_slash_n.txt deleted file mode 100644 index e46edea986..0000000000 --- a/crates/syntax/test_data/lexer/err/0007_unclosed_char_with_slash_n.txt +++ /dev/null @@ -1,2 +0,0 @@ -CHAR 3 "'\\n" -> error0..3 token("'\\n") msg(Missing trailing `'` symbol to terminate the character literal) diff --git a/crates/syntax/test_data/lexer/err/0008_unclosed_char_with_slash_single_quote.txt b/crates/syntax/test_data/lexer/err/0008_unclosed_char_with_slash_single_quote.txt deleted file mode 100644 index 8ad1e913af..0000000000 --- a/crates/syntax/test_data/lexer/err/0008_unclosed_char_with_slash_single_quote.txt +++ /dev/null @@ -1,2 +0,0 @@ -CHAR 3 "'\\'" -> error0..3 token("'\\'") msg(Missing trailing `'` symbol to terminate the character literal) diff --git a/crates/syntax/test_data/lexer/err/0009_unclosed_byte_at_eof.txt b/crates/syntax/test_data/lexer/err/0009_unclosed_byte_at_eof.txt deleted file mode 100644 index 9d30c74669..0000000000 --- a/crates/syntax/test_data/lexer/err/0009_unclosed_byte_at_eof.txt +++ /dev/null @@ -1,2 +0,0 @@ -BYTE 2 "b'" -> error0..2 token("b'") msg(Missing trailing `'` symbol to terminate the byte literal) diff --git a/crates/syntax/test_data/lexer/err/0010_unclosed_byte_with_ferris.txt b/crates/syntax/test_data/lexer/err/0010_unclosed_byte_with_ferris.txt deleted file mode 100644 index 9dbf4203e2..0000000000 --- a/crates/syntax/test_data/lexer/err/0010_unclosed_byte_with_ferris.txt +++ /dev/null @@ -1,2 +0,0 @@ -BYTE 6 "b'π¦" -> error0..6 token("b'π¦") msg(Missing trailing `'` symbol to terminate the byte literal) diff --git a/crates/syntax/test_data/lexer/err/0011_unclosed_byte_with_ascii_escape.txt b/crates/syntax/test_data/lexer/err/0011_unclosed_byte_with_ascii_escape.txt deleted file mode 100644 index d5d9c2ef70..0000000000 --- a/crates/syntax/test_data/lexer/err/0011_unclosed_byte_with_ascii_escape.txt +++ /dev/null @@ -1,2 +0,0 @@ -BYTE 6 "b'\\x7f" -> error0..6 token("b'\\x7f") msg(Missing trailing `'` symbol to terminate the byte literal) diff --git a/crates/syntax/test_data/lexer/err/0012_unclosed_byte_with_unicode_escape.txt b/crates/syntax/test_data/lexer/err/0012_unclosed_byte_with_unicode_escape.txt deleted file mode 100644 index a99b9666a9..0000000000 --- a/crates/syntax/test_data/lexer/err/0012_unclosed_byte_with_unicode_escape.txt +++ /dev/null @@ -1,2 +0,0 @@ -BYTE 10 "b'\\u{20AA}" -> error0..10 token("b'\\u{20AA}") msg(Missing trailing `'` symbol to terminate the byte literal) diff --git a/crates/syntax/test_data/lexer/err/0013_unclosed_byte_with_space.txt b/crates/syntax/test_data/lexer/err/0013_unclosed_byte_with_space.txt deleted file mode 100644 index 8a344f7129..0000000000 --- a/crates/syntax/test_data/lexer/err/0013_unclosed_byte_with_space.txt +++ /dev/null @@ -1,2 +0,0 @@ -BYTE 3 "b' " -> error0..3 token("b' ") msg(Missing trailing `'` symbol to terminate the byte literal) diff --git a/crates/syntax/test_data/lexer/err/0014_unclosed_byte_with_slash.txt b/crates/syntax/test_data/lexer/err/0014_unclosed_byte_with_slash.txt deleted file mode 100644 index b78a43c027..0000000000 --- a/crates/syntax/test_data/lexer/err/0014_unclosed_byte_with_slash.txt +++ /dev/null @@ -1,2 +0,0 @@ -BYTE 3 "b'\\" -> error0..3 token("b'\\") msg(Missing trailing `'` symbol to terminate the byte literal) diff --git a/crates/syntax/test_data/lexer/err/0015_unclosed_byte_with_slash_n.txt b/crates/syntax/test_data/lexer/err/0015_unclosed_byte_with_slash_n.txt deleted file mode 100644 index 5147363ba4..0000000000 --- a/crates/syntax/test_data/lexer/err/0015_unclosed_byte_with_slash_n.txt +++ /dev/null @@ -1,2 +0,0 @@ -BYTE 4 "b'\\n" -> error0..4 token("b'\\n") msg(Missing trailing `'` symbol to terminate the byte literal) diff --git a/crates/syntax/test_data/lexer/err/0016_unclosed_byte_with_slash_single_quote.txt b/crates/syntax/test_data/lexer/err/0016_unclosed_byte_with_slash_single_quote.txt deleted file mode 100644 index 261c0894f0..0000000000 --- a/crates/syntax/test_data/lexer/err/0016_unclosed_byte_with_slash_single_quote.txt +++ /dev/null @@ -1,2 +0,0 @@ -BYTE 4 "b'\\'" -> error0..4 token("b'\\'") msg(Missing trailing `'` symbol to terminate the byte literal) diff --git a/crates/syntax/test_data/lexer/err/0017_unclosed_string_at_eof.txt b/crates/syntax/test_data/lexer/err/0017_unclosed_string_at_eof.txt deleted file mode 100644 index d11a8d880e..0000000000 --- a/crates/syntax/test_data/lexer/err/0017_unclosed_string_at_eof.txt +++ /dev/null @@ -1,2 +0,0 @@ -STRING 1 "\"" -> error0..1 token("\"") msg(Missing trailing `"` symbol to terminate the string literal) diff --git a/crates/syntax/test_data/lexer/err/0018_unclosed_string_with_ferris.txt b/crates/syntax/test_data/lexer/err/0018_unclosed_string_with_ferris.txt deleted file mode 100644 index 167f942d12..0000000000 --- a/crates/syntax/test_data/lexer/err/0018_unclosed_string_with_ferris.txt +++ /dev/null @@ -1,2 +0,0 @@ -STRING 5 "\"π¦" -> error0..5 token("\"π¦") msg(Missing trailing `"` symbol to terminate the string literal) diff --git a/crates/syntax/test_data/lexer/err/0019_unclosed_string_with_ascii_escape.txt b/crates/syntax/test_data/lexer/err/0019_unclosed_string_with_ascii_escape.txt deleted file mode 100644 index 224c653d26..0000000000 --- a/crates/syntax/test_data/lexer/err/0019_unclosed_string_with_ascii_escape.txt +++ /dev/null @@ -1,2 +0,0 @@ -STRING 5 "\"\\x7f" -> error0..5 token("\"\\x7f") msg(Missing trailing `"` symbol to terminate the string literal) diff --git a/crates/syntax/test_data/lexer/err/0020_unclosed_string_with_unicode_escape.txt b/crates/syntax/test_data/lexer/err/0020_unclosed_string_with_unicode_escape.txt deleted file mode 100644 index 48975bbcbc..0000000000 --- a/crates/syntax/test_data/lexer/err/0020_unclosed_string_with_unicode_escape.txt +++ /dev/null @@ -1,2 +0,0 @@ -STRING 9 "\"\\u{20AA}" -> error0..9 token("\"\\u{20AA}") msg(Missing trailing `"` symbol to terminate the string literal) diff --git a/crates/syntax/test_data/lexer/err/0021_unclosed_string_with_space.txt b/crates/syntax/test_data/lexer/err/0021_unclosed_string_with_space.txt deleted file mode 100644 index a823cca7a4..0000000000 --- a/crates/syntax/test_data/lexer/err/0021_unclosed_string_with_space.txt +++ /dev/null @@ -1,2 +0,0 @@ -STRING 2 "\" " -> error0..2 token("\" ") msg(Missing trailing `"` symbol to terminate the string literal) diff --git a/crates/syntax/test_data/lexer/err/0022_unclosed_string_with_slash.txt b/crates/syntax/test_data/lexer/err/0022_unclosed_string_with_slash.txt deleted file mode 100644 index 0914f001f5..0000000000 --- a/crates/syntax/test_data/lexer/err/0022_unclosed_string_with_slash.txt +++ /dev/null @@ -1,2 +0,0 @@ -STRING 2 "\"\\" -> error0..2 token("\"\\") msg(Missing trailing `"` symbol to terminate the string literal) diff --git a/crates/syntax/test_data/lexer/err/0023_unclosed_string_with_slash_n.txt b/crates/syntax/test_data/lexer/err/0023_unclosed_string_with_slash_n.txt deleted file mode 100644 index 5674b55fdf..0000000000 --- a/crates/syntax/test_data/lexer/err/0023_unclosed_string_with_slash_n.txt +++ /dev/null @@ -1,2 +0,0 @@ -STRING 3 "\"\\n" -> error0..3 token("\"\\n") msg(Missing trailing `"` symbol to terminate the string literal) diff --git a/crates/syntax/test_data/lexer/err/0024_unclosed_string_with_slash_double_quote.txt b/crates/syntax/test_data/lexer/err/0024_unclosed_string_with_slash_double_quote.txt deleted file mode 100644 index 4c9a774e46..0000000000 --- a/crates/syntax/test_data/lexer/err/0024_unclosed_string_with_slash_double_quote.txt +++ /dev/null @@ -1,2 +0,0 @@ -STRING 3 "\"\\\"" -> error0..3 token("\"\\\"") msg(Missing trailing `"` symbol to terminate the string literal) diff --git a/crates/syntax/test_data/lexer/err/0025_unclosed_byte_string_at_eof.txt b/crates/syntax/test_data/lexer/err/0025_unclosed_byte_string_at_eof.txt deleted file mode 100644 index 04e6b0aa4f..0000000000 --- a/crates/syntax/test_data/lexer/err/0025_unclosed_byte_string_at_eof.txt +++ /dev/null @@ -1,2 +0,0 @@ -BYTE_STRING 2 "b\"" -> error0..2 token("b\"") msg(Missing trailing `"` symbol to terminate the byte string literal) diff --git a/crates/syntax/test_data/lexer/err/0026_unclosed_byte_string_with_ferris.txt b/crates/syntax/test_data/lexer/err/0026_unclosed_byte_string_with_ferris.txt deleted file mode 100644 index 0576a0609a..0000000000 --- a/crates/syntax/test_data/lexer/err/0026_unclosed_byte_string_with_ferris.txt +++ /dev/null @@ -1,2 +0,0 @@ -BYTE_STRING 6 "b\"π¦" -> error0..6 token("b\"π¦") msg(Missing trailing `"` symbol to terminate the byte string literal) diff --git a/crates/syntax/test_data/lexer/err/0027_unclosed_byte_string_with_ascii_escape.txt b/crates/syntax/test_data/lexer/err/0027_unclosed_byte_string_with_ascii_escape.txt deleted file mode 100644 index 541a013d85..0000000000 --- a/crates/syntax/test_data/lexer/err/0027_unclosed_byte_string_with_ascii_escape.txt +++ /dev/null @@ -1,2 +0,0 @@ -BYTE_STRING 6 "b\"\\x7f" -> error0..6 token("b\"\\x7f") msg(Missing trailing `"` symbol to terminate the byte string literal) diff --git a/crates/syntax/test_data/lexer/err/0028_unclosed_byte_string_with_unicode_escape.txt b/crates/syntax/test_data/lexer/err/0028_unclosed_byte_string_with_unicode_escape.txt deleted file mode 100644 index 71b0fb2118..0000000000 --- a/crates/syntax/test_data/lexer/err/0028_unclosed_byte_string_with_unicode_escape.txt +++ /dev/null @@ -1,2 +0,0 @@ -BYTE_STRING 10 "b\"\\u{20AA}" -> error0..10 token("b\"\\u{20AA}") msg(Missing trailing `"` symbol to terminate the byte string literal) diff --git a/crates/syntax/test_data/lexer/err/0029_unclosed_byte_string_with_space.txt b/crates/syntax/test_data/lexer/err/0029_unclosed_byte_string_with_space.txt deleted file mode 100644 index bd5058bc11..0000000000 --- a/crates/syntax/test_data/lexer/err/0029_unclosed_byte_string_with_space.txt +++ /dev/null @@ -1,2 +0,0 @@ -BYTE_STRING 3 "b\" " -> error0..3 token("b\" ") msg(Missing trailing `"` symbol to terminate the byte string literal) diff --git a/crates/syntax/test_data/lexer/err/0030_unclosed_byte_string_with_slash.txt b/crates/syntax/test_data/lexer/err/0030_unclosed_byte_string_with_slash.txt deleted file mode 100644 index 7f94f10ba1..0000000000 --- a/crates/syntax/test_data/lexer/err/0030_unclosed_byte_string_with_slash.txt +++ /dev/null @@ -1,2 +0,0 @@ -BYTE_STRING 3 "b\"\\" -> error0..3 token("b\"\\") msg(Missing trailing `"` symbol to terminate the byte string literal) diff --git a/crates/syntax/test_data/lexer/err/0031_unclosed_byte_string_with_slash_n.txt b/crates/syntax/test_data/lexer/err/0031_unclosed_byte_string_with_slash_n.txt deleted file mode 100644 index 9c3c089d72..0000000000 --- a/crates/syntax/test_data/lexer/err/0031_unclosed_byte_string_with_slash_n.txt +++ /dev/null @@ -1,2 +0,0 @@ -BYTE_STRING 4 "b\"\\n" -> error0..4 token("b\"\\n") msg(Missing trailing `"` symbol to terminate the byte string literal) diff --git a/crates/syntax/test_data/lexer/err/0032_unclosed_byte_string_with_slash_double_quote.txt b/crates/syntax/test_data/lexer/err/0032_unclosed_byte_string_with_slash_double_quote.txt deleted file mode 100644 index 884b12c8ee..0000000000 --- a/crates/syntax/test_data/lexer/err/0032_unclosed_byte_string_with_slash_double_quote.txt +++ /dev/null @@ -1,2 +0,0 @@ -BYTE_STRING 4 "b\"\\\"" -> error0..4 token("b\"\\\"") msg(Missing trailing `"` symbol to terminate the byte string literal) diff --git a/crates/syntax/test_data/lexer/err/0033_unclosed_raw_string_at_eof.txt b/crates/syntax/test_data/lexer/err/0033_unclosed_raw_string_at_eof.txt deleted file mode 100644 index 54e707b736..0000000000 --- a/crates/syntax/test_data/lexer/err/0033_unclosed_raw_string_at_eof.txt +++ /dev/null @@ -1,2 +0,0 @@ -STRING 4 "r##\"" -> error0..4 token("r##\"") msg(Missing trailing `"` with `#` symbols to terminate the raw string literal) diff --git a/crates/syntax/test_data/lexer/err/0034_unclosed_raw_string_with_ferris.txt b/crates/syntax/test_data/lexer/err/0034_unclosed_raw_string_with_ferris.txt deleted file mode 100644 index 1f9889775f..0000000000 --- a/crates/syntax/test_data/lexer/err/0034_unclosed_raw_string_with_ferris.txt +++ /dev/null @@ -1,2 +0,0 @@ -STRING 8 "r##\"π¦" -> error0..8 token("r##\"π¦") msg(Missing trailing `"` with `#` symbols to terminate the raw string literal) diff --git a/crates/syntax/test_data/lexer/err/0035_unclosed_raw_string_with_ascii_escape.txt b/crates/syntax/test_data/lexer/err/0035_unclosed_raw_string_with_ascii_escape.txt deleted file mode 100644 index 93f6f72ae6..0000000000 --- a/crates/syntax/test_data/lexer/err/0035_unclosed_raw_string_with_ascii_escape.txt +++ /dev/null @@ -1,2 +0,0 @@ -STRING 8 "r##\"\\x7f" -> error0..8 token("r##\"\\x7f") msg(Missing trailing `"` with `#` symbols to terminate the raw string literal) diff --git a/crates/syntax/test_data/lexer/err/0036_unclosed_raw_string_with_unicode_escape.txt b/crates/syntax/test_data/lexer/err/0036_unclosed_raw_string_with_unicode_escape.txt deleted file mode 100644 index 1d2ebc60fd..0000000000 --- a/crates/syntax/test_data/lexer/err/0036_unclosed_raw_string_with_unicode_escape.txt +++ /dev/null @@ -1,2 +0,0 @@ -STRING 12 "r##\"\\u{20AA}" -> error0..12 token("r##\"\\u{20AA}") msg(Missing trailing `"` with `#` symbols to terminate the raw string literal) diff --git a/crates/syntax/test_data/lexer/err/0037_unclosed_raw_string_with_space.txt b/crates/syntax/test_data/lexer/err/0037_unclosed_raw_string_with_space.txt deleted file mode 100644 index c567ab7e28..0000000000 --- a/crates/syntax/test_data/lexer/err/0037_unclosed_raw_string_with_space.txt +++ /dev/null @@ -1,2 +0,0 @@ -STRING 5 "r##\" " -> error0..5 token("r##\" ") msg(Missing trailing `"` with `#` symbols to terminate the raw string literal) diff --git a/crates/syntax/test_data/lexer/err/0038_unclosed_raw_string_with_slash.txt b/crates/syntax/test_data/lexer/err/0038_unclosed_raw_string_with_slash.txt deleted file mode 100644 index 343b20323a..0000000000 --- a/crates/syntax/test_data/lexer/err/0038_unclosed_raw_string_with_slash.txt +++ /dev/null @@ -1,2 +0,0 @@ -STRING 5 "r##\"\\" -> error0..5 token("r##\"\\") msg(Missing trailing `"` with `#` symbols to terminate the raw string literal) diff --git a/crates/syntax/test_data/lexer/err/0039_unclosed_raw_string_with_slash_n.txt b/crates/syntax/test_data/lexer/err/0039_unclosed_raw_string_with_slash_n.txt deleted file mode 100644 index 041a427375..0000000000 --- a/crates/syntax/test_data/lexer/err/0039_unclosed_raw_string_with_slash_n.txt +++ /dev/null @@ -1,2 +0,0 @@ -STRING 6 "r##\"\\n" -> error0..6 token("r##\"\\n") msg(Missing trailing `"` with `#` symbols to terminate the raw string literal) diff --git a/crates/syntax/test_data/lexer/err/0040_unclosed_raw_byte_string_at_eof.txt b/crates/syntax/test_data/lexer/err/0040_unclosed_raw_byte_string_at_eof.txt deleted file mode 100644 index efaa1cafd9..0000000000 --- a/crates/syntax/test_data/lexer/err/0040_unclosed_raw_byte_string_at_eof.txt +++ /dev/null @@ -1,2 +0,0 @@ -BYTE_STRING 5 "br##\"" -> error0..5 token("br##\"") msg(Missing trailing `"` with `#` symbols to terminate the raw byte string literal) diff --git a/crates/syntax/test_data/lexer/err/0041_unclosed_raw_byte_string_with_ferris.txt b/crates/syntax/test_data/lexer/err/0041_unclosed_raw_byte_string_with_ferris.txt deleted file mode 100644 index b6c938f94e..0000000000 --- a/crates/syntax/test_data/lexer/err/0041_unclosed_raw_byte_string_with_ferris.txt +++ /dev/null @@ -1,2 +0,0 @@ -BYTE_STRING 9 "br##\"π¦" -> error0..9 token("br##\"π¦") msg(Missing trailing `"` with `#` symbols to terminate the raw byte string literal) diff --git a/crates/syntax/test_data/lexer/err/0042_unclosed_raw_byte_string_with_ascii_escape.txt b/crates/syntax/test_data/lexer/err/0042_unclosed_raw_byte_string_with_ascii_escape.txt deleted file mode 100644 index f82efe49ac..0000000000 --- a/crates/syntax/test_data/lexer/err/0042_unclosed_raw_byte_string_with_ascii_escape.txt +++ /dev/null @@ -1,2 +0,0 @@ -BYTE_STRING 9 "br##\"\\x7f" -> error0..9 token("br##\"\\x7f") msg(Missing trailing `"` with `#` symbols to terminate the raw byte string literal) diff --git a/crates/syntax/test_data/lexer/err/0043_unclosed_raw_byte_string_with_unicode_escape.txt b/crates/syntax/test_data/lexer/err/0043_unclosed_raw_byte_string_with_unicode_escape.txt deleted file mode 100644 index 4e4a576961..0000000000 --- a/crates/syntax/test_data/lexer/err/0043_unclosed_raw_byte_string_with_unicode_escape.txt +++ /dev/null @@ -1,2 +0,0 @@ -BYTE_STRING 13 "br##\"\\u{20AA}" -> error0..13 token("br##\"\\u{20AA}") msg(Missing trailing `"` with `#` symbols to terminate the raw byte string literal) diff --git a/crates/syntax/test_data/lexer/err/0044_unclosed_raw_byte_string_with_space.txt b/crates/syntax/test_data/lexer/err/0044_unclosed_raw_byte_string_with_space.txt deleted file mode 100644 index 0018c8623c..0000000000 --- a/crates/syntax/test_data/lexer/err/0044_unclosed_raw_byte_string_with_space.txt +++ /dev/null @@ -1,2 +0,0 @@ -BYTE_STRING 6 "br##\" " -> error0..6 token("br##\" ") msg(Missing trailing `"` with `#` symbols to terminate the raw byte string literal) diff --git a/crates/syntax/test_data/lexer/err/0045_unclosed_raw_byte_string_with_slash.txt b/crates/syntax/test_data/lexer/err/0045_unclosed_raw_byte_string_with_slash.txt deleted file mode 100644 index c3ba4ae82b..0000000000 --- a/crates/syntax/test_data/lexer/err/0045_unclosed_raw_byte_string_with_slash.txt +++ /dev/null @@ -1,2 +0,0 @@ -BYTE_STRING 6 "br##\"\\" -> error0..6 token("br##\"\\") msg(Missing trailing `"` with `#` symbols to terminate the raw byte string literal) diff --git a/crates/syntax/test_data/lexer/err/0046_unclosed_raw_byte_string_with_slash_n.txt b/crates/syntax/test_data/lexer/err/0046_unclosed_raw_byte_string_with_slash_n.txt deleted file mode 100644 index 7bda72276f..0000000000 --- a/crates/syntax/test_data/lexer/err/0046_unclosed_raw_byte_string_with_slash_n.txt +++ /dev/null @@ -1,2 +0,0 @@ -BYTE_STRING 7 "br##\"\\n" -> error0..7 token("br##\"\\n") msg(Missing trailing `"` with `#` symbols to terminate the raw byte string literal) diff --git a/crates/syntax/test_data/lexer/err/0047_unstarted_raw_string_at_eof.txt b/crates/syntax/test_data/lexer/err/0047_unstarted_raw_string_at_eof.txt deleted file mode 100644 index ce92d2ff75..0000000000 --- a/crates/syntax/test_data/lexer/err/0047_unstarted_raw_string_at_eof.txt +++ /dev/null @@ -1,2 +0,0 @@ -STRING 3 "r##" -> error0..3 token("r##") msg(Missing `"` symbol after `#` symbols to begin the raw string literal) diff --git a/crates/syntax/test_data/lexer/err/0048_unstarted_raw_byte_string_at_eof.txt b/crates/syntax/test_data/lexer/err/0048_unstarted_raw_byte_string_at_eof.txt deleted file mode 100644 index a75d9030c4..0000000000 --- a/crates/syntax/test_data/lexer/err/0048_unstarted_raw_byte_string_at_eof.txt +++ /dev/null @@ -1,2 +0,0 @@ -BYTE_STRING 4 "br##" -> error0..4 token("br##") msg(Missing `"` symbol after `#` symbols to begin the raw byte string literal) diff --git a/crates/syntax/test_data/lexer/err/0049_unstarted_raw_string_with_ascii.txt b/crates/syntax/test_data/lexer/err/0049_unstarted_raw_string_with_ascii.txt deleted file mode 100644 index 516e0b78ed..0000000000 --- a/crates/syntax/test_data/lexer/err/0049_unstarted_raw_string_with_ascii.txt +++ /dev/null @@ -1,10 +0,0 @@ -STRING 4 "r## " -IDENT 1 "I" -WHITESPACE 1 " " -IDENT 4 "lack" -WHITESPACE 1 " " -IDENT 1 "a" -WHITESPACE 1 " " -IDENT 5 "quote" -BANG 1 "!" -> error0..4 token("r## ") msg(Missing `"` symbol after `#` symbols to begin the raw string literal) diff --git a/crates/syntax/test_data/lexer/err/0050_unstarted_raw_byte_string_with_ascii.txt b/crates/syntax/test_data/lexer/err/0050_unstarted_raw_byte_string_with_ascii.txt deleted file mode 100644 index 2f8a6f5f29..0000000000 --- a/crates/syntax/test_data/lexer/err/0050_unstarted_raw_byte_string_with_ascii.txt +++ /dev/null @@ -1,10 +0,0 @@ -BYTE_STRING 5 "br## " -IDENT 1 "I" -WHITESPACE 1 " " -IDENT 4 "lack" -WHITESPACE 1 " " -IDENT 1 "a" -WHITESPACE 1 " " -IDENT 5 "quote" -BANG 1 "!" -> error0..5 token("br## ") msg(Missing `"` symbol after `#` symbols to begin the raw byte string literal) diff --git a/crates/syntax/test_data/lexer/err/0051_unclosed_block_comment_at_eof.txt b/crates/syntax/test_data/lexer/err/0051_unclosed_block_comment_at_eof.txt deleted file mode 100644 index 2a256e9dff..0000000000 --- a/crates/syntax/test_data/lexer/err/0051_unclosed_block_comment_at_eof.txt +++ /dev/null @@ -1,2 +0,0 @@ -COMMENT 2 "/*" -> error0..2 token("/*") msg(Missing trailing `*/` symbols to terminate the block comment) diff --git a/crates/syntax/test_data/lexer/err/0052_unclosed_block_comment_with_content.txt b/crates/syntax/test_data/lexer/err/0052_unclosed_block_comment_with_content.txt deleted file mode 100644 index 8e8490302a..0000000000 --- a/crates/syntax/test_data/lexer/err/0052_unclosed_block_comment_with_content.txt +++ /dev/null @@ -1,2 +0,0 @@ -COMMENT 11 "/* comment\n" -> error0..11 token("/* comment\n") msg(Missing trailing `*/` symbols to terminate the block comment) diff --git a/crates/syntax/test_data/lexer/err/0053_unclosed_nested_block_comment_entirely.txt b/crates/syntax/test_data/lexer/err/0053_unclosed_nested_block_comment_entirely.txt deleted file mode 100644 index b7d28fc05d..0000000000 --- a/crates/syntax/test_data/lexer/err/0053_unclosed_nested_block_comment_entirely.txt +++ /dev/null @@ -1,2 +0,0 @@ -COMMENT 9 "/* /* /*\n" -> error0..9 token("/* /* /*\n") msg(Missing trailing `*/` symbols to terminate the block comment) diff --git a/crates/syntax/test_data/lexer/err/0054_unclosed_nested_block_comment_partially.txt b/crates/syntax/test_data/lexer/err/0054_unclosed_nested_block_comment_partially.txt deleted file mode 100644 index 4742d2c12c..0000000000 --- a/crates/syntax/test_data/lexer/err/0054_unclosed_nested_block_comment_partially.txt +++ /dev/null @@ -1,2 +0,0 @@ -COMMENT 25 "/** /*! /* comment */ */\n" -> error0..25 token("/** /*! /* comment */ */\n") msg(Missing trailing `*/` symbols to terminate the block comment) diff --git a/crates/syntax/test_data/lexer/err/0055_empty_int.txt b/crates/syntax/test_data/lexer/err/0055_empty_int.txt deleted file mode 100644 index bcd094b6ef..0000000000 --- a/crates/syntax/test_data/lexer/err/0055_empty_int.txt +++ /dev/null @@ -1,39 +0,0 @@ -INT_NUMBER 2 "0b" -WHITESPACE 1 "\n" -INT_NUMBER 2 "0o" -WHITESPACE 1 "\n" -INT_NUMBER 2 "0x" -WHITESPACE 2 "\n\n" -INT_NUMBER 3 "0b_" -WHITESPACE 1 "\n" -INT_NUMBER 3 "0o_" -WHITESPACE 1 "\n" -INT_NUMBER 3 "0x_" -WHITESPACE 2 "\n\n" -INT_NUMBER 9 "0bnoDigit" -WHITESPACE 1 "\n" -INT_NUMBER 9 "0onoDigit" -WHITESPACE 1 "\n" -INT_NUMBER 9 "0xnoDigit" -WHITESPACE 2 "\n\n" -INT_NUMBER 3 "0xG" -WHITESPACE 1 "\n" -INT_NUMBER 3 "0xg" -WHITESPACE 2 "\n\n" -INT_NUMBER 4 "0x_g" -WHITESPACE 1 "\n" -INT_NUMBER 4 "0x_G" -WHITESPACE 1 "\n" -> error0..2 token("0b") msg(Missing digits after the integer base prefix) -> error3..5 token("0o") msg(Missing digits after the integer base prefix) -> error6..8 token("0x") msg(Missing digits after the integer base prefix) -> error10..13 token("0b_") msg(Missing digits after the integer base prefix) -> error14..17 token("0o_") msg(Missing digits after the integer base prefix) -> error18..21 token("0x_") msg(Missing digits after the integer base prefix) -> error23..32 token("0bnoDigit") msg(Missing digits after the integer base prefix) -> error33..42 token("0onoDigit") msg(Missing digits after the integer base prefix) -> error43..52 token("0xnoDigit") msg(Missing digits after the integer base prefix) -> error54..57 token("0xG") msg(Missing digits after the integer base prefix) -> error58..61 token("0xg") msg(Missing digits after the integer base prefix) -> error63..67 token("0x_g") msg(Missing digits after the integer base prefix) -> error68..72 token("0x_G") msg(Missing digits after the integer base prefix) diff --git a/crates/syntax/test_data/lexer/err/0056_empty_exponent.txt b/crates/syntax/test_data/lexer/err/0056_empty_exponent.txt deleted file mode 100644 index 6a645a6a44..0000000000 --- a/crates/syntax/test_data/lexer/err/0056_empty_exponent.txt +++ /dev/null @@ -1,62 +0,0 @@ -FLOAT_NUMBER 2 "0e" -WHITESPACE 1 "\n" -FLOAT_NUMBER 2 "0E" -WHITESPACE 2 "\n\n" -FLOAT_NUMBER 4 "42e+" -WHITESPACE 1 "\n" -FLOAT_NUMBER 4 "42e-" -WHITESPACE 1 "\n" -FLOAT_NUMBER 4 "42E+" -WHITESPACE 1 "\n" -FLOAT_NUMBER 4 "42E-" -WHITESPACE 2 "\n\n" -INT_NUMBER 2 "42" -DOT 1 "." -IDENT 1 "e" -PLUS 1 "+" -WHITESPACE 1 "\n" -INT_NUMBER 2 "42" -DOT 1 "." -IDENT 1 "e" -MINUS 1 "-" -WHITESPACE 1 "\n" -INT_NUMBER 2 "42" -DOT 1 "." -IDENT 1 "E" -PLUS 1 "+" -WHITESPACE 1 "\n" -INT_NUMBER 2 "42" -DOT 1 "." -IDENT 1 "E" -MINUS 1 "-" -WHITESPACE 2 "\n\n" -FLOAT_NUMBER 6 "42.2e+" -WHITESPACE 1 "\n" -FLOAT_NUMBER 6 "42.2e-" -WHITESPACE 1 "\n" -FLOAT_NUMBER 6 "42.2E+" -WHITESPACE 1 "\n" -FLOAT_NUMBER 6 "42.2E-" -WHITESPACE 2 "\n\n" -FLOAT_NUMBER 9 "42.2e+f32" -WHITESPACE 1 "\n" -FLOAT_NUMBER 9 "42.2e-f32" -WHITESPACE 1 "\n" -FLOAT_NUMBER 9 "42.2E+f32" -WHITESPACE 1 "\n" -FLOAT_NUMBER 9 "42.2E-f32" -WHITESPACE 1 "\n" -> error0..2 token("0e") msg(Missing digits after the exponent symbol) -> error3..5 token("0E") msg(Missing digits after the exponent symbol) -> error7..11 token("42e+") msg(Missing digits after the exponent symbol) -> error12..16 token("42e-") msg(Missing digits after the exponent symbol) -> error17..21 token("42E+") msg(Missing digits after the exponent symbol) -> error22..26 token("42E-") msg(Missing digits after the exponent symbol) -> error53..59 token("42.2e+") msg(Missing digits after the exponent symbol) -> error60..66 token("42.2e-") msg(Missing digits after the exponent symbol) -> error67..73 token("42.2E+") msg(Missing digits after the exponent symbol) -> error74..80 token("42.2E-") msg(Missing digits after the exponent symbol) -> error82..91 token("42.2e+f32") msg(Missing digits after the exponent symbol) -> error92..101 token("42.2e-f32") msg(Missing digits after the exponent symbol) -> error102..111 token("42.2E+f32") msg(Missing digits after the exponent symbol) -> error112..121 token("42.2E-f32") msg(Missing digits after the exponent symbol) diff --git a/crates/syntax/test_data/lexer/err/0057_lifetime_starts_with_a_number.txt b/crates/syntax/test_data/lexer/err/0057_lifetime_starts_with_a_number.txt deleted file mode 100644 index b746404d22..0000000000 --- a/crates/syntax/test_data/lexer/err/0057_lifetime_starts_with_a_number.txt +++ /dev/null @@ -1,6 +0,0 @@ -LIFETIME_IDENT 2 "'1" -WHITESPACE 1 "\n" -LIFETIME_IDENT 10 "'1lifetime" -WHITESPACE 1 "\n" -> error0..2 token("'1") msg(Lifetime name cannot start with a number) -> error3..13 token("'1lifetime") msg(Lifetime name cannot start with a number) diff --git a/crates/syntax/test_data/lexer/ok/0001_hello.txt b/crates/syntax/test_data/lexer/ok/0001_hello.txt deleted file mode 100644 index 27a5940a9c..0000000000 --- a/crates/syntax/test_data/lexer/ok/0001_hello.txt +++ /dev/null @@ -1,3 +0,0 @@ -IDENT 5 "hello" -WHITESPACE 1 " " -IDENT 5 "world" diff --git a/crates/syntax/test_data/lexer/ok/0002_whitespace.txt b/crates/syntax/test_data/lexer/ok/0002_whitespace.txt deleted file mode 100644 index 01d260918e..0000000000 --- a/crates/syntax/test_data/lexer/ok/0002_whitespace.txt +++ /dev/null @@ -1,12 +0,0 @@ -IDENT 1 "a" -WHITESPACE 1 " " -IDENT 1 "b" -WHITESPACE 2 " " -IDENT 1 "c" -WHITESPACE 1 "\n" -IDENT 1 "d" -WHITESPACE 2 "\n\n" -IDENT 1 "e" -WHITESPACE 1 "\t" -IDENT 1 "f" -WHITESPACE 1 "\n" diff --git a/crates/syntax/test_data/lexer/ok/0003_ident.txt b/crates/syntax/test_data/lexer/ok/0003_ident.txt deleted file mode 100644 index 4a0d5c0531..0000000000 --- a/crates/syntax/test_data/lexer/ok/0003_ident.txt +++ /dev/null @@ -1,14 +0,0 @@ -IDENT 3 "foo" -WHITESPACE 1 " " -IDENT 4 "foo_" -WHITESPACE 1 " " -IDENT 4 "_foo" -WHITESPACE 1 " " -UNDERSCORE 1 "_" -WHITESPACE 1 " " -IDENT 2 "__" -WHITESPACE 1 " " -IDENT 1 "x" -WHITESPACE 1 " " -IDENT 12 "ΠΏΡΠΈΠ²Π΅Ρ" -WHITESPACE 1 "\n" diff --git a/crates/syntax/test_data/lexer/ok/0004_numbers.txt b/crates/syntax/test_data/lexer/ok/0004_numbers.txt deleted file mode 100644 index e19fc5789e..0000000000 --- a/crates/syntax/test_data/lexer/ok/0004_numbers.txt +++ /dev/null @@ -1,57 +0,0 @@ -INT_NUMBER 1 "0" -WHITESPACE 1 " " -INT_NUMBER 2 "00" -WHITESPACE 1 " " -INT_NUMBER 2 "0_" -WHITESPACE 1 " " -FLOAT_NUMBER 2 "0." -WHITESPACE 1 " " -INT_NUMBER 2 "0z" -WHITESPACE 1 "\n" -INT_NUMBER 5 "01790" -WHITESPACE 1 " " -INT_NUMBER 6 "0b1790" -WHITESPACE 1 " " -INT_NUMBER 6 "0o1790" -WHITESPACE 1 " " -INT_NUMBER 18 "0x1790aAbBcCdDeEfF" -WHITESPACE 1 " " -INT_NUMBER 6 "001279" -WHITESPACE 1 " " -INT_NUMBER 6 "0_1279" -WHITESPACE 1 " " -FLOAT_NUMBER 6 "0.1279" -WHITESPACE 1 " " -FLOAT_NUMBER 6 "0e1279" -WHITESPACE 1 " " -FLOAT_NUMBER 6 "0E1279" -WHITESPACE 1 "\n" -INT_NUMBER 1 "0" -DOT 1 "." -DOT 1 "." -INT_NUMBER 1 "2" -WHITESPACE 1 "\n" -INT_NUMBER 1 "0" -DOT 1 "." -IDENT 3 "foo" -L_PAREN 1 "(" -R_PAREN 1 ")" -WHITESPACE 1 "\n" -FLOAT_NUMBER 4 "0e+1" -WHITESPACE 1 "\n" -INT_NUMBER 1 "0" -DOT 1 "." -IDENT 1 "e" -PLUS 1 "+" -INT_NUMBER 1 "1" -WHITESPACE 1 "\n" -FLOAT_NUMBER 6 "0.0E-2" -WHITESPACE 1 "\n" -FLOAT_NUMBER 26 "0___0.10000____0000e+111__" -WHITESPACE 1 "\n" -INT_NUMBER 4 "1i64" -WHITESPACE 1 " " -FLOAT_NUMBER 7 "92.0f32" -WHITESPACE 1 " " -INT_NUMBER 5 "11__s" -WHITESPACE 1 "\n" diff --git a/crates/syntax/test_data/lexer/ok/0005_symbols.txt b/crates/syntax/test_data/lexer/ok/0005_symbols.txt deleted file mode 100644 index 2049c2f1d6..0000000000 --- a/crates/syntax/test_data/lexer/ok/0005_symbols.txt +++ /dev/null @@ -1,77 +0,0 @@ -SEMICOLON 1 ";" -WHITESPACE 1 " " -COMMA 1 "," -WHITESPACE 1 " " -L_PAREN 1 "(" -WHITESPACE 1 " " -R_PAREN 1 ")" -WHITESPACE 1 " " -L_CURLY 1 "{" -WHITESPACE 1 " " -R_CURLY 1 "}" -WHITESPACE 1 " " -L_BRACK 1 "[" -WHITESPACE 1 " " -R_BRACK 1 "]" -WHITESPACE 1 " " -L_ANGLE 1 "<" -WHITESPACE 1 " " -R_ANGLE 1 ">" -WHITESPACE 1 " " -AT 1 "@" -WHITESPACE 1 " " -POUND 1 "#" -WHITESPACE 1 " " -TILDE 1 "~" -WHITESPACE 1 " " -QUESTION 1 "?" -WHITESPACE 1 " " -DOLLAR 1 "$" -WHITESPACE 1 " " -AMP 1 "&" -WHITESPACE 1 " " -PIPE 1 "|" -WHITESPACE 1 " " -PLUS 1 "+" -WHITESPACE 1 " " -STAR 1 "*" -WHITESPACE 1 " " -SLASH 1 "/" -WHITESPACE 1 " " -CARET 1 "^" -WHITESPACE 1 " " -PERCENT 1 "%" -WHITESPACE 1 "\n" -DOT 1 "." -WHITESPACE 1 " " -DOT 1 "." -DOT 1 "." -WHITESPACE 1 " " -DOT 1 "." -DOT 1 "." -DOT 1 "." -WHITESPACE 1 " " -DOT 1 "." -DOT 1 "." -EQ 1 "=" -WHITESPACE 1 "\n" -COLON 1 ":" -WHITESPACE 1 " " -COLON 1 ":" -COLON 1 ":" -WHITESPACE 1 "\n" -EQ 1 "=" -WHITESPACE 1 " " -EQ 1 "=" -R_ANGLE 1 ">" -WHITESPACE 1 "\n" -BANG 1 "!" -WHITESPACE 1 " " -BANG 1 "!" -EQ 1 "=" -WHITESPACE 1 "\n" -MINUS 1 "-" -WHITESPACE 1 " " -MINUS 1 "-" -R_ANGLE 1 ">" -WHITESPACE 1 "\n" diff --git a/crates/syntax/test_data/lexer/ok/0006_chars.txt b/crates/syntax/test_data/lexer/ok/0006_chars.txt deleted file mode 100644 index 756477dc91..0000000000 --- a/crates/syntax/test_data/lexer/ok/0006_chars.txt +++ /dev/null @@ -1,16 +0,0 @@ -CHAR 3 "'x'" -WHITESPACE 1 " " -CHAR 3 "' '" -WHITESPACE 1 " " -CHAR 3 "'0'" -WHITESPACE 1 " " -CHAR 7 "'hello'" -WHITESPACE 1 " " -CHAR 6 "'\\x7f'" -WHITESPACE 1 " " -CHAR 4 "'\\n'" -WHITESPACE 1 " " -CHAR 4 "'\\\\'" -WHITESPACE 1 " " -CHAR 4 "'\\''" -WHITESPACE 1 "\n" diff --git a/crates/syntax/test_data/lexer/ok/0007_lifetimes.txt b/crates/syntax/test_data/lexer/ok/0007_lifetimes.txt deleted file mode 100644 index 32ed9ed506..0000000000 --- a/crates/syntax/test_data/lexer/ok/0007_lifetimes.txt +++ /dev/null @@ -1,8 +0,0 @@ -LIFETIME_IDENT 2 "'a" -WHITESPACE 1 " " -LIFETIME_IDENT 4 "'foo" -WHITESPACE 1 " " -LIFETIME_IDENT 12 "'foo_bar_baz" -WHITESPACE 1 " " -LIFETIME_IDENT 2 "'_" -WHITESPACE 1 "\n" diff --git a/crates/syntax/test_data/lexer/ok/0008_byte_strings.txt b/crates/syntax/test_data/lexer/ok/0008_byte_strings.txt deleted file mode 100644 index 06d6bdd1f9..0000000000 --- a/crates/syntax/test_data/lexer/ok/0008_byte_strings.txt +++ /dev/null @@ -1,22 +0,0 @@ -BYTE 3 "b''" -WHITESPACE 1 " " -BYTE 4 "b'x'" -WHITESPACE 1 " " -BYTE_STRING 6 "b\"foo\"" -WHITESPACE 1 " " -BYTE_STRING 4 "br\"\"" -WHITESPACE 1 "\n" -BYTE 6 "b''suf" -WHITESPACE 1 " " -BYTE_STRING 5 "b\"\"ix" -WHITESPACE 1 " " -BYTE_STRING 6 "br\"\"br" -WHITESPACE 1 "\n" -BYTE 5 "b'\\n'" -WHITESPACE 1 " " -BYTE 5 "b'\\\\'" -WHITESPACE 1 " " -BYTE 5 "b'\\''" -WHITESPACE 1 " " -BYTE 8 "b'hello'" -WHITESPACE 1 "\n" diff --git a/crates/syntax/test_data/lexer/ok/0009_strings.txt b/crates/syntax/test_data/lexer/ok/0009_strings.txt deleted file mode 100644 index 988a8877bd..0000000000 --- a/crates/syntax/test_data/lexer/ok/0009_strings.txt +++ /dev/null @@ -1,8 +0,0 @@ -STRING 7 "\"hello\"" -WHITESPACE 1 " " -STRING 8 "r\"world\"" -WHITESPACE 1 " " -STRING 17 "\"\\n\\\"\\\\no escape\"" -WHITESPACE 1 " " -STRING 12 "\"multi\nline\"" -WHITESPACE 1 "\n" diff --git a/crates/syntax/test_data/lexer/ok/0010_single_line_comments.txt b/crates/syntax/test_data/lexer/ok/0010_single_line_comments.txt deleted file mode 100644 index 98a3818c06..0000000000 --- a/crates/syntax/test_data/lexer/ok/0010_single_line_comments.txt +++ /dev/null @@ -1,22 +0,0 @@ -SHEBANG 19 "#!/usr/bin/env bash" -WHITESPACE 1 "\n" -COMMENT 8 "// hello" -WHITESPACE 1 "\n" -COMMENT 9 "//! World" -WHITESPACE 1 "\n" -COMMENT 19 "//!! Inner line doc" -WHITESPACE 1 "\n" -COMMENT 18 "/// Outer line doc" -WHITESPACE 1 "\n" -COMMENT 19 "//// Just a comment" -WHITESPACE 2 "\n\n" -COMMENT 2 "//" -WHITESPACE 1 "\n" -COMMENT 3 "//!" -WHITESPACE 1 "\n" -COMMENT 4 "//!!" -WHITESPACE 1 "\n" -COMMENT 3 "///" -WHITESPACE 1 "\n" -COMMENT 4 "////" -WHITESPACE 1 "\n" diff --git a/crates/syntax/test_data/lexer/ok/0011_keywords.txt b/crates/syntax/test_data/lexer/ok/0011_keywords.txt deleted file mode 100644 index 22c00eefb6..0000000000 --- a/crates/syntax/test_data/lexer/ok/0011_keywords.txt +++ /dev/null @@ -1,64 +0,0 @@ -ASYNC_KW 5 "async" -WHITESPACE 1 " " -FN_KW 2 "fn" -WHITESPACE 1 " " -USE_KW 3 "use" -WHITESPACE 1 " " -STRUCT_KW 6 "struct" -WHITESPACE 1 " " -TRAIT_KW 5 "trait" -WHITESPACE 1 " " -ENUM_KW 4 "enum" -WHITESPACE 1 " " -IMPL_KW 4 "impl" -WHITESPACE 1 " " -TRUE_KW 4 "true" -WHITESPACE 1 " " -FALSE_KW 5 "false" -WHITESPACE 1 " " -AS_KW 2 "as" -WHITESPACE 1 " " -EXTERN_KW 6 "extern" -WHITESPACE 1 " " -CRATE_KW 5 "crate" -WHITESPACE 1 "\n" -MOD_KW 3 "mod" -WHITESPACE 1 " " -PUB_KW 3 "pub" -WHITESPACE 1 " " -SELF_KW 4 "self" -WHITESPACE 1 " " -SUPER_KW 5 "super" -WHITESPACE 1 " " -IN_KW 2 "in" -WHITESPACE 1 " " -WHERE_KW 5 "where" -WHITESPACE 1 " " -FOR_KW 3 "for" -WHITESPACE 1 " " -LOOP_KW 4 "loop" -WHITESPACE 1 " " -WHILE_KW 5 "while" -WHITESPACE 1 " " -IF_KW 2 "if" -WHITESPACE 1 " " -MATCH_KW 5 "match" -WHITESPACE 1 " " -CONST_KW 5 "const" -WHITESPACE 1 "\n" -STATIC_KW 6 "static" -WHITESPACE 1 " " -MUT_KW 3 "mut" -WHITESPACE 1 " " -TYPE_KW 4 "type" -WHITESPACE 1 " " -REF_KW 3 "ref" -WHITESPACE 1 " " -LET_KW 3 "let" -WHITESPACE 1 " " -ELSE_KW 4 "else" -WHITESPACE 1 " " -MOVE_KW 4 "move" -WHITESPACE 1 " " -RETURN_KW 6 "return" -WHITESPACE 1 "\n" diff --git a/crates/syntax/test_data/lexer/ok/0012_block_comment.txt b/crates/syntax/test_data/lexer/ok/0012_block_comment.txt deleted file mode 100644 index 2618e287e6..0000000000 --- a/crates/syntax/test_data/lexer/ok/0012_block_comment.txt +++ /dev/null @@ -1,6 +0,0 @@ -COMMENT 5 "/* */" -WHITESPACE 1 "\n" -COMMENT 4 "/**/" -WHITESPACE 1 "\n" -COMMENT 11 "/* /* */ */" -WHITESPACE 1 "\n" diff --git a/crates/syntax/test_data/lexer/ok/0013_raw_strings.txt b/crates/syntax/test_data/lexer/ok/0013_raw_strings.txt deleted file mode 100644 index db0d5ffd14..0000000000 --- a/crates/syntax/test_data/lexer/ok/0013_raw_strings.txt +++ /dev/null @@ -1,2 +0,0 @@ -STRING 36 "r###\"this is a r##\"raw\"## string\"###" -WHITESPACE 1 "\n" diff --git a/crates/syntax/test_data/lexer/ok/0014_raw_ident.txt b/crates/syntax/test_data/lexer/ok/0014_raw_ident.txt deleted file mode 100644 index 484689693b..0000000000 --- a/crates/syntax/test_data/lexer/ok/0014_raw_ident.txt +++ /dev/null @@ -1,2 +0,0 @@ -IDENT 11 "r#raw_ident" -WHITESPACE 1 "\n" |