Unnamed repository; edit this file 'description' to name the repository.
Diffstat (limited to 'crates/parser/src/lexed_str.rs')
| -rw-r--r-- | crates/parser/src/lexed_str.rs | 56 |
1 files changed, 28 insertions, 28 deletions
diff --git a/crates/parser/src/lexed_str.rs b/crates/parser/src/lexed_str.rs index 100deff462..e4dce21f32 100644 --- a/crates/parser/src/lexed_str.rs +++ b/crates/parser/src/lexed_str.rs @@ -36,7 +36,7 @@ impl<'a> LexedStr<'a> { }; for token in rustc_lexer::tokenize(&text[conv.offset..]) { - let token_text = &text[conv.offset..][..token.len]; + let token_text = &text[conv.offset..][..token.len as usize]; conv.extend_token(&token.kind, token_text); } @@ -49,8 +49,8 @@ impl<'a> LexedStr<'a> { return None; } - let token = rustc_lexer::first_token(text); - if token.len != text.len() { + let token = rustc_lexer::tokenize(text).next()?; + if token.len as usize != text.len() { return None; } @@ -175,6 +175,10 @@ impl<'a> Converter<'a> { rustc_lexer::TokenKind::Ident => { SyntaxKind::from_keyword(token_text).unwrap_or(IDENT) } + rustc_lexer::TokenKind::InvalidIdent => { + err = "Ident contains invalid characters"; + IDENT + } rustc_lexer::TokenKind::RawIdent => IDENT, rustc_lexer::TokenKind::Literal { kind, .. } => { @@ -221,6 +225,7 @@ impl<'a> Converter<'a> { err = "unknown literal prefix"; IDENT } + rustc_lexer::TokenKind::Eof => EOF, } }; @@ -268,35 +273,30 @@ impl<'a> Converter<'a> { } BYTE_STRING } - rustc_lexer::LiteralKind::RawStr { err: raw_str_err, .. } => { - if let Some(raw_str_err) = raw_str_err { - err = match raw_str_err { - rustc_lexer::RawStrError::InvalidStarter { .. } => "Missing `\"` symbol after `#` symbols to begin the raw string literal", - rustc_lexer::RawStrError::NoTerminator { expected, found, .. } => if expected == found { - "Missing trailing `\"` to terminate the raw string literal" - } else { - "Missing trailing `\"` with `#` symbols to terminate the raw string literal" - }, - rustc_lexer::RawStrError::TooManyDelimiters { .. } => "Too many `#` symbols: raw strings may be delimited by up to 65535 `#` symbols", - }; - }; + rustc_lexer::LiteralKind::CStr { terminated } => { + if !terminated { + err = "Missing trailing `\"` symbol to terminate the string literal"; + } + C_STRING + } + rustc_lexer::LiteralKind::RawStr { n_hashes } => { + if n_hashes.is_none() { + err = "Invalid raw string literal"; + } STRING } - rustc_lexer::LiteralKind::RawByteStr { err: raw_str_err, .. } => { - if let Some(raw_str_err) = raw_str_err { - err = match raw_str_err { - rustc_lexer::RawStrError::InvalidStarter { .. } => "Missing `\"` symbol after `#` symbols to begin the raw byte string literal", - rustc_lexer::RawStrError::NoTerminator { expected, found, .. } => if expected == found { - "Missing trailing `\"` to terminate the raw byte string literal" - } else { - "Missing trailing `\"` with `#` symbols to terminate the raw byte string literal" - }, - rustc_lexer::RawStrError::TooManyDelimiters { .. } => "Too many `#` symbols: raw byte strings may be delimited by up to 65535 `#` symbols", - }; - }; - + rustc_lexer::LiteralKind::RawByteStr { n_hashes } => { + if n_hashes.is_none() { + err = "Invalid raw string literal"; + } BYTE_STRING } + rustc_lexer::LiteralKind::RawCStr { n_hashes } => { + if n_hashes.is_none() { + err = "Invalid raw string literal"; + } + C_STRING + } }; let err = if err.is_empty() { None } else { Some(err) }; |