Unnamed repository; edit this file 'description' to name the repository.
Merge pull request #18417 from ChayimFriedman2/hash-string
fix: Correctly handle `#""` in edition <2024
| -rw-r--r-- | crates/parser/src/lexed_str.rs | 13 | ||||
| -rw-r--r-- | crates/parser/src/tests.rs | 17 | ||||
| -rw-r--r-- | crates/parser/test_data/lexer/ok/guarded_str_prefix_edition_2021.rast | 4 | ||||
| -rw-r--r-- | crates/parser/test_data/lexer/ok/guarded_str_prefix_edition_2021.rs | 3 |
4 files changed, 30 insertions, 7 deletions
diff --git a/crates/parser/src/lexed_str.rs b/crates/parser/src/lexed_str.rs index 5322463a71..3c0eb1b42a 100644 --- a/crates/parser/src/lexed_str.rs +++ b/crates/parser/src/lexed_str.rs @@ -39,7 +39,9 @@ impl<'a> LexedStr<'a> { conv.offset = shebang_len; }; - for token in rustc_lexer::tokenize(&text[conv.offset..]) { + // Re-create the tokenizer from scratch every token because `GuardedStrPrefix` is one token in the lexer + // but we want to split it to two in edition <2024. + while let Some(token) = rustc_lexer::tokenize(&text[conv.offset..]).next() { let token_text = &text[conv.offset..][..token.len as usize]; conv.extend_token(&token.kind, token_text); @@ -158,7 +160,7 @@ impl<'a> Converter<'a> { } } - fn extend_token(&mut self, kind: &rustc_lexer::TokenKind, token_text: &str) { + fn extend_token(&mut self, kind: &rustc_lexer::TokenKind, mut token_text: &str) { // A note on an intended tradeoff: // We drop some useful information here (see patterns with double dots `..`) // Storing that info in `SyntaxKind` is not possible due to its layout requirements of @@ -189,10 +191,15 @@ impl<'a> Converter<'a> { rustc_lexer::TokenKind::RawIdent => IDENT, rustc_lexer::TokenKind::GuardedStrPrefix if self.edition.at_least_2024() => { + // FIXME: rustc does something better for recovery. err = "Invalid string literal (reserved syntax)"; ERROR } - rustc_lexer::TokenKind::GuardedStrPrefix => POUND, + rustc_lexer::TokenKind::GuardedStrPrefix => { + // The token is `#"` or `##`, split it into two. + token_text = &token_text[1..]; + POUND + } rustc_lexer::TokenKind::Literal { kind, .. } => { self.extend_literal(token_text.len(), kind); diff --git a/crates/parser/src/tests.rs b/crates/parser/src/tests.rs index e7bccb6685..4b19ddc752 100644 --- a/crates/parser/src/tests.rs +++ b/crates/parser/src/tests.rs @@ -15,11 +15,20 @@ use crate::{Edition, LexedStr, TopEntryPoint}; #[path = "../test_data/generated/runner.rs"] mod runner; +fn infer_edition(file_path: &Path) -> Edition { + let file_content = std::fs::read_to_string(file_path).unwrap(); + if let Some(edition) = file_content.strip_prefix("//@ edition: ") { + edition[..4].parse().expect("invalid edition directive") + } else { + Edition::CURRENT + } +} + #[test] fn lex_ok() { for case in TestCase::list("lexer/ok") { let _guard = stdx::panic_context::enter(format!("{:?}", case.rs)); - let actual = lex(&case.text); + let actual = lex(&case.text, infer_edition(&case.rs)); expect_file![case.rast].assert_eq(&actual) } } @@ -28,13 +37,13 @@ fn lex_ok() { fn lex_err() { for case in TestCase::list("lexer/err") { let _guard = stdx::panic_context::enter(format!("{:?}", case.rs)); - let actual = lex(&case.text); + let actual = lex(&case.text, infer_edition(&case.rs)); expect_file![case.rast].assert_eq(&actual) } } -fn lex(text: &str) -> String { - let lexed = LexedStr::new(Edition::CURRENT, text); +fn lex(text: &str, edition: Edition) -> String { + let lexed = LexedStr::new(edition, text); let mut res = String::new(); for i in 0..lexed.len() { diff --git a/crates/parser/test_data/lexer/ok/guarded_str_prefix_edition_2021.rast b/crates/parser/test_data/lexer/ok/guarded_str_prefix_edition_2021.rast new file mode 100644 index 0000000000..1bdd672044 --- /dev/null +++ b/crates/parser/test_data/lexer/ok/guarded_str_prefix_edition_2021.rast @@ -0,0 +1,4 @@ +COMMENT "//@ edition: 2021" +WHITESPACE "\n\n" +POUND "#" +STRING "\"foo\"" diff --git a/crates/parser/test_data/lexer/ok/guarded_str_prefix_edition_2021.rs b/crates/parser/test_data/lexer/ok/guarded_str_prefix_edition_2021.rs new file mode 100644 index 0000000000..f00f949f0d --- /dev/null +++ b/crates/parser/test_data/lexer/ok/guarded_str_prefix_edition_2021.rs @@ -0,0 +1,3 @@ +//@ edition: 2021 + +#"foo"
\ No newline at end of file |