Unnamed repository; edit this file 'description' to name the repository.
Auto merge of #12241 - jonas-schievink:does-not-float-my-boat, r=jonas-schievink
fix: revert float parsing "fix" to avoid macro-related panics
Reverts https://github.com/rust-lang/rust-analyzer/pull/12149 and the follow-up fixes, while keeping their tests.
https://github.com/rust-lang/rust-analyzer/pull/12149 has caused many unexpected panics related to macros, and the fixes for those are not straightforward and further complicate the MBE token conversion logic, which was already fairly hard to follow before these fixes.
42 files changed, 152 insertions, 606 deletions
diff --git a/crates/hir-def/src/body/lower.rs b/crates/hir-def/src/body/lower.rs index 84662aa277..e8303ec40f 100644 --- a/crates/hir-def/src/body/lower.rs +++ b/crates/hir-def/src/body/lower.rs @@ -972,7 +972,7 @@ impl From<ast::LiteralKind> for Literal { } } LiteralKind::FloatNumber(lit) => { - let ty = lit.suffix().and_then(|s| BuiltinFloat::from_suffix(&s)); + let ty = lit.suffix().and_then(BuiltinFloat::from_suffix); Literal::Float(Default::default(), ty) } LiteralKind::ByteString(bs) => { diff --git a/crates/hir-def/src/macro_expansion_tests/mbe.rs b/crates/hir-def/src/macro_expansion_tests/mbe.rs index 8e01920483..befef6547c 100644 --- a/crates/hir-def/src/macro_expansion_tests/mbe.rs +++ b/crates/hir-def/src/macro_expansion_tests/mbe.rs @@ -48,6 +48,8 @@ struct#10 MyTraitMap2#32 {#13 #[test] fn token_mapping_floats() { + // Regression test for https://github.com/rust-lang/rust-analyzer/issues/12216 + // (and related issues) check( r#" // +tokenids @@ -87,9 +89,9 @@ macro_rules! f {#0 // } fn#19 main#20(#21)#21 {#22 1#23;#24 - 1#26.0; - let x#31 =#22 1; -} + 1.0#25;#26 + let#27 x#28 =#29 1#30;#31 +}#22 "##]], diff --git a/crates/hir-def/src/macro_expansion_tests/mbe/meta_syntax.rs b/crates/hir-def/src/macro_expansion_tests/mbe/meta_syntax.rs index 66cc2843d6..2de10ddbdf 100644 --- a/crates/hir-def/src/macro_expansion_tests/mbe/meta_syntax.rs +++ b/crates/hir-def/src/macro_expansion_tests/mbe/meta_syntax.rs @@ -80,7 +80,7 @@ macro_rules! f3 { ($i:_) => () } #[test] fn test_rustc_issue_57597() { - // <https://github.com/rust-lang/rust/blob/master/src/test/ui/macros/issue-57597.rs> + // <https://github.com/rust-lang/rust/blob/master/src/test/ui/issues/issue-57597.rs> check( r#" macro_rules! m0 { ($($($i:ident)?)+) => {}; } diff --git a/crates/hir-def/src/macro_expansion_tests/mbe/tt_conversion.rs b/crates/hir-def/src/macro_expansion_tests/mbe/tt_conversion.rs index 9e4ab043f6..0710b1ac3d 100644 --- a/crates/hir-def/src/macro_expansion_tests/mbe/tt_conversion.rs +++ b/crates/hir-def/src/macro_expansion_tests/mbe/tt_conversion.rs @@ -38,7 +38,6 @@ macro_rules! m { let _ = 12E+99_f64; let _ = "rust1"; let _ = -92; - let _ = -1.3e4f32; } } fn f() { @@ -53,7 +52,6 @@ macro_rules! m { let _ = 12E+99_f64; let _ = "rust1"; let _ = -92; - let _ = -1.3e4f32; } } fn f() { @@ -62,7 +60,6 @@ fn f() { let _ = 12E+99_f64; let _ = "rust1"; let _ = -92; - let _ = -1.3e4f32; } "#]], ); @@ -153,55 +150,51 @@ $ = (); } #[test] -fn float_literal_in_output() { +fn float_literal_in_tt() { check( r#" macro_rules! constant { - ($e:expr ;) => {$e}; + ($( $ret:expr; )*) => {}; } - -const _: () = constant!(0.0;); -const _: () = constant!(0.;); -const _: () = constant!(0e0;); +macro_rules! float_const_impl { + () => ( constant!(0.3; 3.3;); ); +} +float_const_impl! {} "#, expect![[r#" macro_rules! constant { - ($e:expr ;) => {$e}; + ($( $ret:expr; )*) => {}; } - -const _: () = 0.0; -const _: () = 0.; -const _: () = 0e0; +macro_rules! float_const_impl { + () => ( constant!(0.3; 3.3;); ); +} +constant!(0.3; +3.3; +); "#]], ); } #[test] -fn float_literal_in_tt() { +fn float_literal_in_output() { check( r#" macro_rules! constant { - ($( $ret:expr; )*) => {}; -} - -macro_rules! float_const_impl { - () => ( constant!(0.3; 3.3;); ); + ($e:expr ;) => {$e}; } -float_const_impl! {} +const _: () = constant!(0.0;); +const _: () = constant!(0.;); +const _: () = constant!(0e0;); "#, expect![[r#" macro_rules! constant { - ($( $ret:expr; )*) => {}; -} - -macro_rules! float_const_impl { - () => ( constant!(0.3; 3.3;); ); + ($e:expr ;) => {$e}; } -constant!(0.3; -3.3; -); +const _: () = 0.0; +const _: () = 0.; +const _: () = 0e0; "#]], ); } diff --git a/crates/hir-def/src/macro_expansion_tests/proc_macros.rs b/crates/hir-def/src/macro_expansion_tests/proc_macros.rs index e4b065d020..72c44a0fbc 100644 --- a/crates/hir-def/src/macro_expansion_tests/proc_macros.rs +++ b/crates/hir-def/src/macro_expansion_tests/proc_macros.rs @@ -104,8 +104,7 @@ macro_rules! id { $($t)* }; } - -id! { +id /*+errors*/! { #[proc_macros::identity] impl Foo for WrapBj { async fn foo(&self) { @@ -120,7 +119,7 @@ macro_rules! id { $($t)* }; } - +/* parse error: expected SEMICOLON */ #[proc_macros::identity] impl Foo for WrapBj { async fn foo(&self ) { self .0.id().await ; diff --git a/crates/hir-expand/src/builtin_fn_macro.rs b/crates/hir-expand/src/builtin_fn_macro.rs index 2be6c78a6e..aa09912f30 100644 --- a/crates/hir-expand/src/builtin_fn_macro.rs +++ b/crates/hir-expand/src/builtin_fn_macro.rs @@ -4,7 +4,10 @@ use base_db::{AnchoredPath, Edition, FileId}; use cfg::CfgExpr; use either::Either; use mbe::{parse_exprs_with_sep, parse_to_token_tree}; -use syntax::{ast, SmolStr}; +use syntax::{ + ast::{self, AstToken}, + SmolStr, +}; use crate::{db::AstDatabase, name, quote, ExpandError, ExpandResult, MacroCallId, MacroCallLoc}; @@ -355,7 +358,14 @@ fn unreachable_expand( } fn unquote_str(lit: &tt::Literal) -> Option<String> { - let token = ast::make::literal(&lit.to_string()).as_string()?; + let lit = ast::make::tokens::literal(&lit.to_string()); + let token = ast::String::cast(lit)?; + token.value().map(|it| it.into_owned()) +} + +fn unquote_byte_string(lit: &tt::Literal) -> Option<Vec<u8>> { + let lit = ast::make::tokens::literal(&lit.to_string()); + let token = ast::ByteString::cast(lit)?; token.value().map(|it| it.into_owned()) } @@ -432,16 +442,12 @@ fn concat_bytes_expand( for (i, t) in tt.token_trees.iter().enumerate() { match t { tt::TokenTree::Leaf(tt::Leaf::Literal(lit)) => { - let lit = ast::make::literal(&lit.to_string()); - match lit.kind() { - ast::LiteralKind::ByteString(s) => { - s.value() - .unwrap_or_default() - .into_iter() - .for_each(|x| bytes.push(x.to_string())); - } - ast::LiteralKind::Byte(_) => { - bytes.push(lit.to_string()); + let token = ast::make::tokens::literal(&lit.to_string()); + match token.kind() { + syntax::SyntaxKind::BYTE => bytes.push(token.text().to_string()), + syntax::SyntaxKind::BYTE_STRING => { + let components = unquote_byte_string(lit).unwrap_or_else(Vec::new); + components.into_iter().for_each(|x| bytes.push(x.to_string())); } _ => { err.get_or_insert(mbe::ExpandError::UnexpectedToken.into()); @@ -475,10 +481,10 @@ fn concat_bytes_expand_subtree( for (ti, tt) in tree.token_trees.iter().enumerate() { match tt { tt::TokenTree::Leaf(tt::Leaf::Literal(lit)) => { - let lit = ast::make::literal(&lit.to_string()); + let lit = ast::make::tokens::literal(&lit.to_string()); match lit.kind() { - ast::LiteralKind::IntNumber(_) | ast::LiteralKind::Byte(_) => { - bytes.push(lit.to_string()); + syntax::SyntaxKind::BYTE | syntax::SyntaxKind::INT_NUMBER => { + bytes.push(lit.text().to_string()) } _ => { return Err(mbe::ExpandError::UnexpectedToken.into()); diff --git a/crates/hir-ty/src/tests/simple.rs b/crates/hir-ty/src/tests/simple.rs index 4b8a7e782d..d4d61c2167 100644 --- a/crates/hir-ty/src/tests/simple.rs +++ b/crates/hir-ty/src/tests/simple.rs @@ -2733,14 +2733,3 @@ fn f() { "#, ); } - -#[test] -fn nested_tuple_index() { - check_no_mismatches( - r#" -fn main() { - let fld: i32 = ((0,),).0.0; -} -"#, - ); -} diff --git a/crates/ide-completion/src/completions/dot.rs b/crates/ide-completion/src/completions/dot.rs index 6a553eadc1..9b5ececfd0 100644 --- a/crates/ide-completion/src/completions/dot.rs +++ b/crates/ide-completion/src/completions/dot.rs @@ -793,24 +793,4 @@ fn main() { ", ) } - - #[test] - fn tuple_index_completion() { - check( - r#" -struct I; -impl I { - fn i_method(&self) {} -} -struct S((), I); - -fn f(s: S) { - s.1.$0 -} -"#, - expect![[r#" - me i_method() fn(&self) - "#]], - ); - } } diff --git a/crates/ide-completion/src/completions/postfix.rs b/crates/ide-completion/src/completions/postfix.rs index be0f674889..28256bd704 100644 --- a/crates/ide-completion/src/completions/postfix.rs +++ b/crates/ide-completion/src/completions/postfix.rs @@ -5,7 +5,7 @@ mod format_like; use hir::{Documentation, HasAttrs}; use ide_db::{imports::insert_use::ImportScope, ty_filter::TryEnum, SnippetCap}; use syntax::{ - ast::{self, AstNode, LiteralKind}, + ast::{self, AstNode, AstToken}, SyntaxKind::{EXPR_STMT, STMT_LIST}, TextRange, TextSize, }; @@ -194,7 +194,7 @@ pub(crate) fn complete_postfix(acc: &mut Completions, ctx: &CompletionContext) { } if let ast::Expr::Literal(literal) = dot_receiver.clone() { - if let LiteralKind::String(literal_text) = literal.kind() { + if let Some(literal_text) = ast::String::cast(literal.token()) { add_format_like_completions(acc, ctx, &dot_receiver, cap, &literal_text); } } diff --git a/crates/ide-completion/src/context.rs b/crates/ide-completion/src/context.rs index 4a058ac261..fbf3278ad1 100644 --- a/crates/ide-completion/src/context.rs +++ b/crates/ide-completion/src/context.rs @@ -1062,7 +1062,7 @@ impl<'a> CompletionContext<'a> { let receiver_is_ambiguous_float_literal = match &receiver { Some(ast::Expr::Literal(l)) => matches! { l.kind(), - ast::LiteralKind::FloatNumber { .. } if l.syntax().last_token().map_or(false, |it| it.kind() == T![.]) + ast::LiteralKind::FloatNumber { .. } if l.syntax().last_token().map_or(false, |it| it.text().ends_with('.')) }, _ => false, }; diff --git a/crates/ide/src/syntax_highlighting/highlight.rs b/crates/ide/src/syntax_highlighting/highlight.rs index bafbf6d9cc..e04fd5a7c7 100644 --- a/crates/ide/src/syntax_highlighting/highlight.rs +++ b/crates/ide/src/syntax_highlighting/highlight.rs @@ -30,15 +30,7 @@ pub(super) fn token(sema: &Semantics<RootDatabase>, token: SyntaxToken) -> Optio INT_NUMBER if token.ancestors().nth(1).map(|it| it.kind()) == Some(FIELD_EXPR) => { SymbolKind::Field.into() } - INT_NUMBER | FLOAT_NUMBER_PART | FLOAT_NUMBER_START_0 | FLOAT_NUMBER_START_1 - | FLOAT_NUMBER_START_2 => HlTag::NumericLiteral.into(), - DOT if matches!( - token.prev_token().map(|n| n.kind()), - Some(FLOAT_NUMBER_START_1 | FLOAT_NUMBER_START_2) - ) => - { - HlTag::NumericLiteral.into() - } + INT_NUMBER | FLOAT_NUMBER => HlTag::NumericLiteral.into(), BYTE => HlTag::ByteLiteral.into(), CHAR => HlTag::CharLiteral.into(), IDENT if token.parent().and_then(ast::TokenTree::cast).is_some() => { diff --git a/crates/ide/src/syntax_highlighting/test_data/highlight_strings.html b/crates/ide/src/syntax_highlighting/test_data/highlight_strings.html index d7d6c79e3d..60bc290121 100644 --- a/crates/ide/src/syntax_highlighting/test_data/highlight_strings.html +++ b/crates/ide/src/syntax_highlighting/test_data/highlight_strings.html @@ -119,13 +119,13 @@ pre { color: #DCDCCC; background: #3F3F3F; font-size: 22px; padd <span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"Hello </span><span class="format_specifier">{</span><span class="format_specifier">:</span><span class="numeric_literal">0</span><span class="numeric_literal">5</span><span class="format_specifier">}</span><span class="string_literal">!"</span><span class="comma">,</span> <span class="numeric_literal">5</span><span class="parenthesis">)</span><span class="semicolon">;</span> <span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"Hello </span><span class="format_specifier">{</span><span class="format_specifier">:</span><span class="numeric_literal">0</span><span class="numeric_literal">5</span><span class="format_specifier">}</span><span class="string_literal">!"</span><span class="comma">,</span> <span class="numeric_literal">-</span><span class="numeric_literal">5</span><span class="parenthesis">)</span><span class="semicolon">;</span> <span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"</span><span class="format_specifier">{</span><span class="format_specifier">:</span><span class="format_specifier">#</span><span class="numeric_literal">0</span><span class="numeric_literal">10</span><span class="variable">x</span><span class="format_specifier">}</span><span class="string_literal">!"</span><span class="comma">,</span> <span class="numeric_literal">27</span><span class="parenthesis">)</span><span class="semicolon">;</span> - <span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"Hello </span><span class="format_specifier">{</span><span class="numeric_literal">0</span><span class="format_specifier">}</span><span class="string_literal"> is </span><span class="format_specifier">{</span><span class="numeric_literal">1</span><span class="format_specifier">:</span><span class="format_specifier">.</span><span class="numeric_literal">5</span><span class="format_specifier">}</span><span class="string_literal">"</span><span class="comma">,</span> <span class="string_literal">"x"</span><span class="comma">,</span> <span class="numeric_literal">0</span><span class="numeric_literal">.</span><span class="numeric_literal">01</span><span class="parenthesis">)</span><span class="semicolon">;</span> - <span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"Hello </span><span class="format_specifier">{</span><span class="numeric_literal">1</span><span class="format_specifier">}</span><span class="string_literal"> is </span><span class="format_specifier">{</span><span class="numeric_literal">2</span><span class="format_specifier">:</span><span class="format_specifier">.</span><span class="numeric_literal">0</span><span class="format_specifier">$</span><span class="format_specifier">}</span><span class="string_literal">"</span><span class="comma">,</span> <span class="numeric_literal">5</span><span class="comma">,</span> <span class="string_literal">"x"</span><span class="comma">,</span> <span class="numeric_literal">0</span><span class="numeric_literal">.</span><span class="numeric_literal">01</span><span class="parenthesis">)</span><span class="semicolon">;</span> - <span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"Hello </span><span class="format_specifier">{</span><span class="numeric_literal">0</span><span class="format_specifier">}</span><span class="string_literal"> is </span><span class="format_specifier">{</span><span class="numeric_literal">2</span><span class="format_specifier">:</span><span class="format_specifier">.</span><span class="numeric_literal">1</span><span class="format_specifier">$</span><span class="format_specifier">}</span><span class="string_literal">"</span><span class="comma">,</span> <span class="string_literal">"x"</span><span class="comma">,</span> <span class="numeric_literal">5</span><span class="comma">,</span> <span class="numeric_literal">0</span><span class="numeric_literal">.</span><span class="numeric_literal">01</span><span class="parenthesis">)</span><span class="semicolon">;</span> - <span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"Hello </span><span class="format_specifier">{</span><span class="format_specifier">}</span><span class="string_literal"> is </span><span class="format_specifier">{</span><span class="format_specifier">:</span><span class="format_specifier">.</span><span class="format_specifier">*</span><span class="format_specifier">}</span><span class="string_literal">"</span><span class="comma">,</span> <span class="string_literal">"x"</span><span class="comma">,</span> <span class="numeric_literal">5</span><span class="comma">,</span> <span class="numeric_literal">0</span><span class="numeric_literal">.</span><span class="numeric_literal">01</span><span class="parenthesis">)</span><span class="semicolon">;</span> - <span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"Hello </span><span class="format_specifier">{</span><span class="format_specifier">}</span><span class="string_literal"> is </span><span class="format_specifier">{</span><span class="numeric_literal">2</span><span class="format_specifier">:</span><span class="format_specifier">.</span><span class="format_specifier">*</span><span class="format_specifier">}</span><span class="string_literal">"</span><span class="comma">,</span> <span class="string_literal">"x"</span><span class="comma">,</span> <span class="numeric_literal">5</span><span class="comma">,</span> <span class="numeric_literal">0</span><span class="numeric_literal">.</span><span class="numeric_literal">01</span><span class="parenthesis">)</span><span class="semicolon">;</span> - <span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"Hello </span><span class="format_specifier">{</span><span class="format_specifier">}</span><span class="string_literal"> is </span><span class="format_specifier">{</span><span class="variable">number</span><span class="format_specifier">:</span><span class="format_specifier">.</span><span class="variable">prec</span><span class="format_specifier">$</span><span class="format_specifier">}</span><span class="string_literal">"</span><span class="comma">,</span> <span class="string_literal">"x"</span><span class="comma">,</span> prec <span class="operator">=</span> <span class="numeric_literal">5</span><span class="comma">,</span> number <span class="operator">=</span> <span class="numeric_literal">0</span><span class="numeric_literal">.</span><span class="numeric_literal">01</span><span class="parenthesis">)</span><span class="semicolon">;</span> - <span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"</span><span class="format_specifier">{</span><span class="format_specifier">}</span><span class="string_literal">, `</span><span class="format_specifier">{</span><span class="variable">name</span><span class="format_specifier">:</span><span class="format_specifier">.</span><span class="format_specifier">*</span><span class="format_specifier">}</span><span class="string_literal">` has 3 fractional digits"</span><span class="comma">,</span> <span class="string_literal">"Hello"</span><span class="comma">,</span> <span class="numeric_literal">3</span><span class="comma">,</span> name<span class="operator">=</span><span class="numeric_literal">1234</span><span class="numeric_literal">.</span><span class="numeric_literal">56</span><span class="parenthesis">)</span><span class="semicolon">;</span> + <span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"Hello </span><span class="format_specifier">{</span><span class="numeric_literal">0</span><span class="format_specifier">}</span><span class="string_literal"> is </span><span class="format_specifier">{</span><span class="numeric_literal">1</span><span class="format_specifier">:</span><span class="format_specifier">.</span><span class="numeric_literal">5</span><span class="format_specifier">}</span><span class="string_literal">"</span><span class="comma">,</span> <span class="string_literal">"x"</span><span class="comma">,</span> <span class="numeric_literal">0.01</span><span class="parenthesis">)</span><span class="semicolon">;</span> + <span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"Hello </span><span class="format_specifier">{</span><span class="numeric_literal">1</span><span class="format_specifier">}</span><span class="string_literal"> is </span><span class="format_specifier">{</span><span class="numeric_literal">2</span><span class="format_specifier">:</span><span class="format_specifier">.</span><span class="numeric_literal">0</span><span class="format_specifier">$</span><span class="format_specifier">}</span><span class="string_literal">"</span><span class="comma">,</span> <span class="numeric_literal">5</span><span class="comma">,</span> <span class="string_literal">"x"</span><span class="comma">,</span> <span class="numeric_literal">0.01</span><span class="parenthesis">)</span><span class="semicolon">;</span> + <span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"Hello </span><span class="format_specifier">{</span><span class="numeric_literal">0</span><span class="format_specifier">}</span><span class="string_literal"> is </span><span class="format_specifier">{</span><span class="numeric_literal">2</span><span class="format_specifier">:</span><span class="format_specifier">.</span><span class="numeric_literal">1</span><span class="format_specifier">$</span><span class="format_specifier">}</span><span class="string_literal">"</span><span class="comma">,</span> <span class="string_literal">"x"</span><span class="comma">,</span> <span class="numeric_literal">5</span><span class="comma">,</span> <span class="numeric_literal">0.01</span><span class="parenthesis">)</span><span class="semicolon">;</span> + <span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"Hello </span><span class="format_specifier">{</span><span class="format_specifier">}</span><span class="string_literal"> is </span><span class="format_specifier">{</span><span class="format_specifier">:</span><span class="format_specifier">.</span><span class="format_specifier">*</span><span class="format_specifier">}</span><span class="string_literal">"</span><span class="comma">,</span> <span class="string_literal">"x"</span><span class="comma">,</span> <span class="numeric_literal">5</span><span class="comma">,</span> <span class="numeric_literal">0.01</span><span class="parenthesis">)</span><span class="semicolon">;</span> + <span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"Hello </span><span class="format_specifier">{</span><span class="format_specifier">}</span><span class="string_literal"> is </span><span class="format_specifier">{</span><span class="numeric_literal">2</span><span class="format_specifier">:</span><span class="format_specifier">.</span><span class="format_specifier">*</span><span class="format_specifier">}</span><span class="string_literal">"</span><span class="comma">,</span> <span class="string_literal">"x"</span><span class="comma">,</span> <span class="numeric_literal">5</span><span class="comma">,</span> <span class="numeric_literal">0.01</span><span class="parenthesis">)</span><span class="semicolon">;</span> + <span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"Hello </span><span class="format_specifier">{</span><span class="format_specifier">}</span><span class="string_literal"> is </span><span class="format_specifier">{</span><span class="variable">number</span><span class="format_specifier">:</span><span class="format_specifier">.</span><span class="variable">prec</span><span class="format_specifier">$</span><span class="format_specifier">}</span><span class="string_literal">"</span><span class="comma">,</span> <span class="string_literal">"x"</span><span class="comma">,</span> prec <span class="operator">=</span> <span class="numeric_literal">5</span><span class="comma">,</span> number <span class="operator">=</span> <span class="numeric_literal">0.01</span><span class="parenthesis">)</span><span class="semicolon">;</span> + <span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"</span><span class="format_specifier">{</span><span class="format_specifier">}</span><span class="string_literal">, `</span><span class="format_specifier">{</span><span class="variable">name</span><span class="format_specifier">:</span><span class="format_specifier">.</span><span class="format_specifier">*</span><span class="format_specifier">}</span><span class="string_literal">` has 3 fractional digits"</span><span class="comma">,</span> <span class="string_literal">"Hello"</span><span class="comma">,</span> <span class="numeric_literal">3</span><span class="comma">,</span> name<span class="operator">=</span><span class="numeric_literal">1234.56</span><span class="parenthesis">)</span><span class="semicolon">;</span> <span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"</span><span class="format_specifier">{</span><span class="format_specifier">}</span><span class="string_literal">, `</span><span class="format_specifier">{</span><span class="variable">name</span><span class="format_specifier">:</span><span class="format_specifier">.</span><span class="format_specifier">*</span><span class="format_specifier">}</span><span class="string_literal">` has 3 characters"</span><span class="comma">,</span> <span class="string_literal">"Hello"</span><span class="comma">,</span> <span class="numeric_literal">3</span><span class="comma">,</span> name<span class="operator">=</span><span class="string_literal">"1234.56"</span><span class="parenthesis">)</span><span class="semicolon">;</span> <span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"</span><span class="format_specifier">{</span><span class="format_specifier">}</span><span class="string_literal">, `</span><span class="format_specifier">{</span><span class="variable">name</span><span class="format_specifier">:</span><span class="format_specifier">></span><span class="numeric_literal">8</span><span class="format_specifier">.</span><span class="format_specifier">*</span><span class="format_specifier">}</span><span class="string_literal">` has 3 right-aligned characters"</span><span class="comma">,</span> <span class="string_literal">"Hello"</span><span class="comma">,</span> <span class="numeric_literal">3</span><span class="comma">,</span> name<span class="operator">=</span><span class="string_literal">"1234.56"</span><span class="parenthesis">)</span><span class="semicolon">;</span> <span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis">(</span><span class="string_literal">"Hello {{}}"</span><span class="parenthesis">)</span><span class="semicolon">;</span> diff --git a/crates/mbe/src/syntax_bridge.rs b/crates/mbe/src/syntax_bridge.rs index 361633e39d..21a0aa4284 100644 --- a/crates/mbe/src/syntax_bridge.rs +++ b/crates/mbe/src/syntax_bridge.rs @@ -243,8 +243,6 @@ fn convert_tokens<C: TokenConvertor>(conv: &mut C) -> tt::Subtree { let char = match token.to_char(conv) { Some(c) => c, None => { - // FIXME: this isn't really correct, `to_char` yields the *first* char of the token, - // and this is relevant when eg. creating 2 `tt::Punct` from a single `::` token panic!("Token from lexer must be single char: token = {:#?}", token); } }; @@ -262,33 +260,6 @@ fn convert_tokens<C: TokenConvertor>(conv: &mut C) -> tt::Subtree { IDENT => make_leaf!(Ident), UNDERSCORE => make_leaf!(Ident), k if k.is_keyword() => make_leaf!(Ident), - FLOAT_NUMBER_START_0 | FLOAT_NUMBER_START_1 | FLOAT_NUMBER_START_2 => { - // Reassemble a split-up float token. - let mut range = range; - let mut text = token.to_text(conv).to_string(); - if kind == FLOAT_NUMBER_START_1 || kind == FLOAT_NUMBER_START_2 { - let (dot, dot_range) = conv.bump().unwrap(); - assert_eq!(dot.kind(conv), DOT); - text += &*dot.to_text(conv); - range = TextRange::new(range.start(), dot_range.end()); - - if kind == FLOAT_NUMBER_START_2 { - let (tail, tail_range) = conv.bump().unwrap(); - assert_eq!(tail.kind(conv), FLOAT_NUMBER_PART); - text += &*tail.to_text(conv); - range = TextRange::new(range.start(), tail_range.end()); - } - } - - result.push( - tt::Leaf::from(tt::Literal { - id: conv.id_alloc().alloc(range, synth_id), - text: text.into(), - }) - .into(), - ); - continue; - } k if k.is_literal() => make_leaf!(Literal), LIFETIME_IDENT => { let char_unit = TextSize::of('\''); @@ -742,7 +713,6 @@ struct TtTreeSink<'a> { text_pos: TextSize, inner: SyntaxTreeBuilder, token_map: TokenMap, - remaining_float_lit_text: String, } impl<'a> TtTreeSink<'a> { @@ -754,7 +724,6 @@ impl<'a> TtTreeSink<'a> { text_pos: 0.into(), inner: SyntaxTreeBuilder::default(), token_map: TokenMap::default(), - remaining_float_lit_text: String::new(), } } @@ -781,54 +750,6 @@ impl<'a> TtTreeSink<'a> { n_tokens = 2; } - // We need to split a float `tt::Literal` into up to 3 tokens consumed by the parser. - match self.cursor.token_tree() { - Some(tt::buffer::TokenTreeRef::Subtree(sub, _)) if sub.delimiter.is_none() => { - self.cursor = self.cursor.subtree().unwrap() - } - _ => {} - } - let literal = match self.cursor.token_tree() { - Some(tt::buffer::TokenTreeRef::Leaf(tt::Leaf::Literal(lit), _)) => Some(lit), - _ => None, - }; - if matches!( - kind, - FLOAT_NUMBER_PART | FLOAT_NUMBER_START_0 | FLOAT_NUMBER_START_1 | FLOAT_NUMBER_START_2 - ) { - if self.remaining_float_lit_text.is_empty() { - always!( - literal.is_some(), - "kind={:?}, cursor tt={:?}", - kind, - self.cursor.token_tree() - ); - let text = literal.map_or(String::new(), |lit| lit.to_string()); - self.cursor = self.cursor.bump(); - match text.split_once('.') { - Some((start, end)) => { - self.inner.token(kind, start); - self.remaining_float_lit_text = format!(".{end}"); - return; - } - None => { - self.inner.token(kind, &text); - return; - } - } - } else { - self.inner.token(kind, &self.remaining_float_lit_text); - self.remaining_float_lit_text.clear(); - return; - } - } - if kind == DOT && !self.remaining_float_lit_text.is_empty() { - always!(self.remaining_float_lit_text.chars().next() == Some('.')); - self.inner.token(kind, "."); - self.remaining_float_lit_text = self.remaining_float_lit_text[1..].to_string(); - return; - } - let mut last = self.cursor; for _ in 0..n_tokens { let tmp: u8; diff --git a/crates/mbe/src/to_parser_input.rs b/crates/mbe/src/to_parser_input.rs index 958f643320..6faa147218 100644 --- a/crates/mbe/src/to_parser_input.rs +++ b/crates/mbe/src/to_parser_input.rs @@ -35,13 +35,15 @@ pub(crate) fn to_parser_input(buffer: &TokenBuffer) -> parser::Input { let is_negated = lit.text.starts_with('-'); let inner_text = &lit.text[if is_negated { 1 } else { 0 }..]; - let lexed_str = parser::LexedStr::new(inner_text); - if lexed_str.is_empty() { - panic!("failed to convert literal: {:?}", lit); - } - for i in 0..lexed_str.len() { - res.push(lexed_str.kind(i)); - } + let kind = parser::LexedStr::single_token(inner_text) + .map(|(kind, _error)| kind) + .filter(|kind| { + kind.is_literal() + && (!is_negated || matches!(kind, FLOAT_NUMBER | INT_NUMBER)) + }) + .unwrap_or_else(|| panic!("Fail to convert given literal {:#?}", &lit)); + + res.push(kind); } tt::Leaf::Ident(ident) => match ident.text.as_ref() { "_" => res.push(T![_]), diff --git a/crates/mbe/src/tt_iter.rs b/crates/mbe/src/tt_iter.rs index 2dbf1f3198..fc5590b718 100644 --- a/crates/mbe/src/tt_iter.rs +++ b/crates/mbe/src/tt_iter.rs @@ -90,20 +90,9 @@ impl<'a> TtIter<'a> { let mut cursor = buffer.begin(); let mut error = false; - let mut float_fragments_to_skip = 0; for step in tree_traversal.iter() { match step { parser::Step::Token { kind, mut n_input_tokens } => { - if float_fragments_to_skip > 0 { - float_fragments_to_skip -= 1; - n_input_tokens = 0; - } - match kind { - SyntaxKind::LIFETIME_IDENT => n_input_tokens = 2, - SyntaxKind::FLOAT_NUMBER_START_1 => float_fragments_to_skip = 1, - SyntaxKind::FLOAT_NUMBER_START_2 => float_fragments_to_skip = 2, - _ => {} - } if kind == SyntaxKind::LIFETIME_IDENT { n_input_tokens = 2; } diff --git a/crates/parser/src/grammar.rs b/crates/parser/src/grammar.rs index 4ebf2157c6..4efbf9a606 100644 --- a/crates/parser/src/grammar.rs +++ b/crates/parser/src/grammar.rs @@ -39,7 +39,6 @@ mod generic_params; mod types; use crate::{ - grammar::expressions::FLOAT_LITERAL_FIRST, parser::{CompletedMarker, Marker, Parser}, SyntaxKind::{self, *}, TokenSet, T, @@ -319,17 +318,9 @@ fn name_ref(p: &mut Parser) { } fn name_ref_or_index(p: &mut Parser) { - assert!( - p.at(IDENT) || p.at(INT_NUMBER) || p.at(FLOAT_NUMBER_PART) || p.at_ts(FLOAT_LITERAL_FIRST) - ); + assert!(p.at(IDENT) || p.at(INT_NUMBER)); let m = p.start(); - if p.at(FLOAT_NUMBER_PART) || p.at_ts(FLOAT_LITERAL_FIRST) { - // Ideally we'd remap this to `INT_NUMBER` instead, but that causes the MBE conversion to - // lose track of what's a float and what isn't, causing panics. - p.bump_remap(FLOAT_NUMBER_PART); - } else { - p.bump_any(); - } + p.bump_any(); m.complete(p, NAME_REF); } diff --git a/crates/parser/src/grammar/expressions.rs b/crates/parser/src/grammar/expressions.rs index 290083b343..b063c73a9d 100644 --- a/crates/parser/src/grammar/expressions.rs +++ b/crates/parser/src/grammar/expressions.rs @@ -3,7 +3,7 @@ mod atom; use super::*; pub(crate) use self::atom::{block_expr, match_arm_list}; -pub(super) use self::atom::{literal, FLOAT_LITERAL_FIRST, LITERAL_FIRST}; +pub(super) use self::atom::{literal, LITERAL_FIRST}; #[derive(PartialEq, Eq)] pub(super) enum Semicolon { @@ -452,9 +452,6 @@ fn index_expr(p: &mut Parser, lhs: CompletedMarker) -> CompletedMarker { // fn foo() { // x.foo(); // y.bar::<T>(1, 2,); -// -// 0e0.sin(); -// 0e0f32.sin(); // } fn method_call_expr(p: &mut Parser, lhs: CompletedMarker) -> CompletedMarker { assert!(p.at(T![.]) && p.nth(1) == IDENT && (p.nth(2) == T!['('] || p.nth_at(2, T![::]))); @@ -472,16 +469,17 @@ fn method_call_expr(p: &mut Parser, lhs: CompletedMarker) -> CompletedMarker { // fn foo() { // x.foo; // x.0.bar; -// x.0. bar; -// x.0.1; // x.0(); // } fn field_expr(p: &mut Parser, lhs: CompletedMarker) -> CompletedMarker { assert!(p.at(T![.])); let m = lhs.precede(p); p.bump(T![.]); - if p.at(IDENT) || p.at(INT_NUMBER) || p.at(FLOAT_NUMBER_PART) || p.at_ts(FLOAT_LITERAL_FIRST) { + if p.at(IDENT) || p.at(INT_NUMBER) { name_ref_or_index(p); + } else if p.at(FLOAT_NUMBER) { + // FIXME: How to recover and instead parse INT + T![.]? + p.bump_any(); } else { p.error("expected field name or number"); } diff --git a/crates/parser/src/grammar/expressions/atom.rs b/crates/parser/src/grammar/expressions/atom.rs index 5ab148ff28..10e5d897e0 100644 --- a/crates/parser/src/grammar/expressions/atom.rs +++ b/crates/parser/src/grammar/expressions/atom.rs @@ -17,58 +17,22 @@ pub(crate) const LITERAL_FIRST: TokenSet = TokenSet::new(&[ T![true], T![false], INT_NUMBER, - FLOAT_NUMBER_START_0, - FLOAT_NUMBER_START_1, - FLOAT_NUMBER_START_2, + FLOAT_NUMBER, BYTE, CHAR, STRING, BYTE_STRING, ]); -pub(crate) const FLOAT_LITERAL_FIRST: TokenSet = - TokenSet::new(&[FLOAT_NUMBER_START_0, FLOAT_NUMBER_START_1, FLOAT_NUMBER_START_2]); - pub(crate) fn literal(p: &mut Parser) -> Option<CompletedMarker> { if !p.at_ts(LITERAL_FIRST) { return None; } let m = p.start(); - if p.at_ts(FLOAT_LITERAL_FIRST) { - float_literal(p); - } else { - // Everything else is just one token. - p.bump_any(); - } + p.bump_any(); Some(m.complete(p, LITERAL)) } -// test float_literal -// fn f() { -// 0.0; -// 1.; -// 0e0; -// 0e0f32; -// 1.23f64; -// } -pub(crate) fn float_literal(p: &mut Parser) { - // Floats can be up to 3 tokens. The first token indicates how many there are. - let f = p.start(); - if p.at(FLOAT_NUMBER_START_0) { - p.bump(FLOAT_NUMBER_START_0); - } else if p.at(FLOAT_NUMBER_START_1) { - p.bump(FLOAT_NUMBER_START_1); - p.bump(DOT); - } else if p.at(FLOAT_NUMBER_START_2) { - p.bump(FLOAT_NUMBER_START_2); - p.bump(DOT); - p.bump(FLOAT_NUMBER_PART); - } else { - unreachable!(); - } - f.complete(p, FLOAT_LITERAL); -} - // E.g. for after the break in `if break {}`, this should not match pub(super) const ATOM_EXPR_FIRST: TokenSet = LITERAL_FIRST.union(paths::PATH_FIRST).union(TokenSet::new(&[ diff --git a/crates/parser/src/grammar/patterns.rs b/crates/parser/src/grammar/patterns.rs index c16bd8d0c7..1f622b32e5 100644 --- a/crates/parser/src/grammar/patterns.rs +++ b/crates/parser/src/grammar/patterns.rs @@ -140,7 +140,7 @@ fn atom_pat(p: &mut Parser, recovery_set: TokenSet) -> Option<CompletedMarker> { } fn is_literal_pat_start(p: &Parser) -> bool { - p.at(T![-]) && (p.nth(1) == INT_NUMBER || p.nth(1) == FLOAT_NUMBER_PART) + p.at(T![-]) && (p.nth(1) == INT_NUMBER || p.nth(1) == FLOAT_NUMBER) || p.at_ts(expressions::LITERAL_FIRST) } diff --git a/crates/parser/src/lexed_str.rs b/crates/parser/src/lexed_str.rs index 9887960871..f4b9988eac 100644 --- a/crates/parser/src/lexed_str.rs +++ b/crates/parser/src/lexed_str.rs @@ -177,7 +177,7 @@ impl<'a> Converter<'a> { rustc_lexer::TokenKind::RawIdent => IDENT, rustc_lexer::TokenKind::Literal { kind, .. } => { - self.extend_literal(token_text, kind); + self.extend_literal(token_text.len(), kind); return; } @@ -223,7 +223,7 @@ impl<'a> Converter<'a> { self.push(syntax_kind, token_text.len(), err); } - fn extend_literal(&mut self, token_text: &str, kind: &rustc_lexer::LiteralKind) { + fn extend_literal(&mut self, len: usize, kind: &rustc_lexer::LiteralKind) { let mut err = ""; let syntax_kind = match *kind { @@ -237,27 +237,7 @@ impl<'a> Converter<'a> { if empty_exponent { err = "Missing digits after the exponent symbol"; } - - // In order to correctly parse nested tuple accesses like `tup.0.0`, where the `0.0` - // is lexed as a float, we split floats that contain a `.` into 3 tokens. - // To ensure that later stages can always reconstruct the token correctly, the first - // token in the sequence indicates the number of following tokens that are part of - // the float literal. - if let Some((before, after)) = token_text.split_once('.') { - let err = if err.is_empty() { None } else { Some(err) }; - - assert!(!before.is_empty()); - let tok = - if after.is_empty() { FLOAT_NUMBER_START_1 } else { FLOAT_NUMBER_START_2 }; - self.push(tok, before.len(), None); - self.push(DOT, 1, None); - if !after.is_empty() { - self.push(FLOAT_NUMBER_PART, after.len(), err); - } - return; - } - - FLOAT_NUMBER_START_0 + FLOAT_NUMBER } rustc_lexer::LiteralKind::Char { terminated } => { if !terminated { @@ -315,6 +295,6 @@ impl<'a> Converter<'a> { }; let err = if err.is_empty() { None } else { Some(err) }; - self.push(syntax_kind, token_text.len(), err); + self.push(syntax_kind, len, err); } } diff --git a/crates/parser/src/syntax_kind/generated.rs b/crates/parser/src/syntax_kind/generated.rs index 5ddcd6ad6a..628fa745e7 100644 --- a/crates/parser/src/syntax_kind/generated.rs +++ b/crates/parser/src/syntax_kind/generated.rs @@ -110,10 +110,7 @@ pub enum SyntaxKind { RAW_KW, MACRO_RULES_KW, INT_NUMBER, - FLOAT_NUMBER_START_0, - FLOAT_NUMBER_START_1, - FLOAT_NUMBER_START_2, - FLOAT_NUMBER_PART, + FLOAT_NUMBER, CHAR, BYTE, STRING, @@ -230,7 +227,6 @@ pub enum SyntaxKind { PATH, PATH_SEGMENT, LITERAL, - FLOAT_LITERAL, RENAME, VISIBILITY, WHERE_CLAUSE, @@ -290,8 +286,7 @@ impl SyntaxKind { } pub fn is_literal(self) -> bool { match self { - INT_NUMBER | FLOAT_NUMBER_START_0 | FLOAT_NUMBER_START_1 | FLOAT_NUMBER_START_2 - | FLOAT_NUMBER_PART | CHAR | BYTE | STRING | BYTE_STRING => true, + INT_NUMBER | FLOAT_NUMBER | CHAR | BYTE | STRING | BYTE_STRING => true, _ => false, } } @@ -391,5 +386,5 @@ impl SyntaxKind { } } #[macro_export] -macro_rules ! T { [;] => { $ crate :: SyntaxKind :: SEMICOLON } ; [,] => { $ crate :: SyntaxKind :: COMMA } ; ['('] => { $ crate :: SyntaxKind :: L_PAREN } ; [')'] => { $ crate :: SyntaxKind :: R_PAREN } ; ['{'] => { $ crate :: SyntaxKind :: L_CURLY } ; ['}'] => { $ crate :: SyntaxKind :: R_CURLY } ; ['['] => { $ crate :: SyntaxKind :: L_BRACK } ; [']'] => { $ crate :: SyntaxKind :: R_BRACK } ; [<] => { $ crate :: SyntaxKind :: L_ANGLE } ; [>] => { $ crate :: SyntaxKind :: R_ANGLE } ; [@] => { $ crate :: SyntaxKind :: AT } ; [#] => { $ crate :: SyntaxKind :: POUND } ; [~] => { $ crate :: SyntaxKind :: TILDE } ; [?] => { $ crate :: SyntaxKind :: QUESTION } ; [$] => { $ crate :: SyntaxKind :: DOLLAR } ; [&] => { $ crate :: SyntaxKind :: AMP } ; [|] => { $ crate :: SyntaxKind :: PIPE } ; [+] => { $ crate :: SyntaxKind :: PLUS } ; [*] => { $ crate :: SyntaxKind :: STAR } ; [/] => { $ crate :: SyntaxKind :: SLASH } ; [^] => { $ crate :: SyntaxKind :: CARET } ; [%] => { $ crate :: SyntaxKind :: PERCENT } ; [_] => { $ crate :: SyntaxKind :: UNDERSCORE } ; [.] => { $ crate :: SyntaxKind :: DOT } ; [..] => { $ crate :: SyntaxKind :: DOT2 } ; [...] => { $ crate :: SyntaxKind :: DOT3 } ; [..=] => { $ crate :: SyntaxKind :: DOT2EQ } ; [:] => { $ crate :: SyntaxKind :: COLON } ; [::] => { $ crate :: SyntaxKind :: COLON2 } ; [=] => { $ crate :: SyntaxKind :: EQ } ; [==] => { $ crate :: SyntaxKind :: EQ2 } ; [=>] => { $ crate :: SyntaxKind :: FAT_ARROW } ; [!] => { $ crate :: SyntaxKind :: BANG } ; [!=] => { $ crate :: SyntaxKind :: NEQ } ; [-] => { $ crate :: SyntaxKind :: MINUS } ; [->] => { $ crate :: SyntaxKind :: THIN_ARROW } ; [<=] => { $ crate :: SyntaxKind :: LTEQ } ; [>=] => { $ crate :: SyntaxKind :: GTEQ } ; [+=] => { $ crate :: SyntaxKind :: PLUSEQ } ; [-=] => { $ crate :: SyntaxKind :: MINUSEQ } ; [|=] => { $ crate :: SyntaxKind :: PIPEEQ } ; [&=] => { $ crate :: SyntaxKind :: AMPEQ } ; [^=] => { $ crate :: SyntaxKind :: CARETEQ } ; [/=] => { $ crate :: SyntaxKind :: SLASHEQ } ; [*=] => { $ crate :: SyntaxKind :: STAREQ } ; [%=] => { $ crate :: SyntaxKind :: PERCENTEQ } ; [&&] => { $ crate :: SyntaxKind :: AMP2 } ; [||] => { $ crate :: SyntaxKind :: PIPE2 } ; [<<] => { $ crate :: SyntaxKind :: SHL } ; [>>] => { $ crate :: SyntaxKind :: SHR } ; [<<=] => { $ crate :: SyntaxKind :: SHLEQ } ; [>>=] => { $ crate :: SyntaxKind :: SHREQ } ; [as] => { $ crate :: SyntaxKind :: AS_KW } ; [async] => { $ crate :: SyntaxKind :: ASYNC_KW } ; [await] => { $ crate :: SyntaxKind :: AWAIT_KW } ; [box] => { $ crate :: SyntaxKind :: BOX_KW } ; [break] => { $ crate :: SyntaxKind :: BREAK_KW } ; [const] => { $ crate :: SyntaxKind :: CONST_KW } ; [continue] => { $ crate :: SyntaxKind :: CONTINUE_KW } ; [crate] => { $ crate :: SyntaxKind :: CRATE_KW } ; [dyn] => { $ crate :: SyntaxKind :: DYN_KW } ; [else] => { $ crate :: SyntaxKind :: ELSE_KW } ; [enum] => { $ crate :: SyntaxKind :: ENUM_KW } ; [extern] => { $ crate :: SyntaxKind :: EXTERN_KW } ; [false] => { $ crate :: SyntaxKind :: FALSE_KW } ; [fn] => { $ crate :: SyntaxKind :: FN_KW } ; [for] => { $ crate :: SyntaxKind :: FOR_KW } ; [if] => { $ crate :: SyntaxKind :: IF_KW } ; [impl] => { $ crate :: SyntaxKind :: IMPL_KW } ; [in] => { $ crate :: SyntaxKind :: IN_KW } ; [let] => { $ crate :: SyntaxKind :: LET_KW } ; [loop] => { $ crate :: SyntaxKind :: LOOP_KW } ; [macro] => { $ crate :: SyntaxKind :: MACRO_KW } ; [match] => { $ crate :: SyntaxKind :: MATCH_KW } ; [mod] => { $ crate :: SyntaxKind :: MOD_KW } ; [move] => { $ crate :: SyntaxKind :: MOVE_KW } ; [mut] => { $ crate :: SyntaxKind :: MUT_KW } ; [pub] => { $ crate :: SyntaxKind :: PUB_KW } ; [ref] => { $ crate :: SyntaxKind :: REF_KW } ; [return] => { $ crate :: SyntaxKind :: RETURN_KW } ; [self] => { $ crate :: SyntaxKind :: SELF_KW } ; [Self] => { $ crate :: SyntaxKind :: SELF_TYPE_KW } ; [static] => { $ crate :: SyntaxKind :: STATIC_KW } ; [struct] => { $ crate :: SyntaxKind :: STRUCT_KW } ; [super] => { $ crate :: SyntaxKind :: SUPER_KW } ; [trait] => { $ crate :: SyntaxKind :: TRAIT_KW } ; [true] => { $ crate :: SyntaxKind :: TRUE_KW } ; [try] => { $ crate :: SyntaxKind :: TRY_KW } ; [type] => { $ crate :: SyntaxKind :: TYPE_KW } ; [unsafe] => { $ crate :: SyntaxKind :: UNSAFE_KW } ; [use] => { $ crate :: SyntaxKind :: USE_KW } ; [where] => { $ crate :: SyntaxKind :: WHERE_KW } ; [while] => { $ crate :: SyntaxKind :: WHILE_KW } ; [yield] => { $ crate :: SyntaxKind :: YIELD_KW } ; [auto] => { $ crate :: SyntaxKind :: AUTO_KW } ; [default] => { $ crate :: SyntaxKind :: DEFAULT_KW } ; [existential] => { $ crate :: SyntaxKind :: EXISTENTIAL_KW } ; [union] => { $ crate :: SyntaxKind :: UNION_KW } ; [raw] => { $ crate :: SyntaxKind :: RAW_KW } ; [macro_rules] => { $ crate :: SyntaxKind :: MACRO_RULES_KW } ; [lifetime_ident] => { $ crate :: SyntaxKind :: LIFETIME_IDENT } ; [ident] => { $ crate :: SyntaxKind :: IDENT } ; [shebang] => { $ crate :: SyntaxKind :: SHEBANG } ; [float_number_part] => { $ crate :: SyntaxKind :: FLOAT_NUMBER_PART } ; [float_number_start_0] => { $ crate :: SyntaxKind :: FLOAT_NUMBER_START_0 } ; [float_number_start_1] => { $ crate :: SyntaxKind :: FLOAT_NUMBER_START_1 } ; [float_number_start_2] => { $ crate :: SyntaxKind :: FLOAT_NUMBER_START_2 } ; } +macro_rules ! T { [;] => { $ crate :: SyntaxKind :: SEMICOLON } ; [,] => { $ crate :: SyntaxKind :: COMMA } ; ['('] => { $ crate :: SyntaxKind :: L_PAREN } ; [')'] => { $ crate :: SyntaxKind :: R_PAREN } ; ['{'] => { $ crate :: SyntaxKind :: L_CURLY } ; ['}'] => { $ crate :: SyntaxKind :: R_CURLY } ; ['['] => { $ crate :: SyntaxKind :: L_BRACK } ; [']'] => { $ crate :: SyntaxKind :: R_BRACK } ; [<] => { $ crate :: SyntaxKind :: L_ANGLE } ; [>] => { $ crate :: SyntaxKind :: R_ANGLE } ; [@] => { $ crate :: SyntaxKind :: AT } ; [#] => { $ crate :: SyntaxKind :: POUND } ; [~] => { $ crate :: SyntaxKind :: TILDE } ; [?] => { $ crate :: SyntaxKind :: QUESTION } ; [$] => { $ crate :: SyntaxKind :: DOLLAR } ; [&] => { $ crate :: SyntaxKind :: AMP } ; [|] => { $ crate :: SyntaxKind :: PIPE } ; [+] => { $ crate :: SyntaxKind :: PLUS } ; [*] => { $ crate :: SyntaxKind :: STAR } ; [/] => { $ crate :: SyntaxKind :: SLASH } ; [^] => { $ crate :: SyntaxKind :: CARET } ; [%] => { $ crate :: SyntaxKind :: PERCENT } ; [_] => { $ crate :: SyntaxKind :: UNDERSCORE } ; [.] => { $ crate :: SyntaxKind :: DOT } ; [..] => { $ crate :: SyntaxKind :: DOT2 } ; [...] => { $ crate :: SyntaxKind :: DOT3 } ; [..=] => { $ crate :: SyntaxKind :: DOT2EQ } ; [:] => { $ crate :: SyntaxKind :: COLON } ; [::] => { $ crate :: SyntaxKind :: COLON2 } ; [=] => { $ crate :: SyntaxKind :: EQ } ; [==] => { $ crate :: SyntaxKind :: EQ2 } ; [=>] => { $ crate :: SyntaxKind :: FAT_ARROW } ; [!] => { $ crate :: SyntaxKind :: BANG } ; [!=] => { $ crate :: SyntaxKind :: NEQ } ; [-] => { $ crate :: SyntaxKind :: MINUS } ; [->] => { $ crate :: SyntaxKind :: THIN_ARROW } ; [<=] => { $ crate :: SyntaxKind :: LTEQ } ; [>=] => { $ crate :: SyntaxKind :: GTEQ } ; [+=] => { $ crate :: SyntaxKind :: PLUSEQ } ; [-=] => { $ crate :: SyntaxKind :: MINUSEQ } ; [|=] => { $ crate :: SyntaxKind :: PIPEEQ } ; [&=] => { $ crate :: SyntaxKind :: AMPEQ } ; [^=] => { $ crate :: SyntaxKind :: CARETEQ } ; [/=] => { $ crate :: SyntaxKind :: SLASHEQ } ; [*=] => { $ crate :: SyntaxKind :: STAREQ } ; [%=] => { $ crate :: SyntaxKind :: PERCENTEQ } ; [&&] => { $ crate :: SyntaxKind :: AMP2 } ; [||] => { $ crate :: SyntaxKind :: PIPE2 } ; [<<] => { $ crate :: SyntaxKind :: SHL } ; [>>] => { $ crate :: SyntaxKind :: SHR } ; [<<=] => { $ crate :: SyntaxKind :: SHLEQ } ; [>>=] => { $ crate :: SyntaxKind :: SHREQ } ; [as] => { $ crate :: SyntaxKind :: AS_KW } ; [async] => { $ crate :: SyntaxKind :: ASYNC_KW } ; [await] => { $ crate :: SyntaxKind :: AWAIT_KW } ; [box] => { $ crate :: SyntaxKind :: BOX_KW } ; [break] => { $ crate :: SyntaxKind :: BREAK_KW } ; [const] => { $ crate :: SyntaxKind :: CONST_KW } ; [continue] => { $ crate :: SyntaxKind :: CONTINUE_KW } ; [crate] => { $ crate :: SyntaxKind :: CRATE_KW } ; [dyn] => { $ crate :: SyntaxKind :: DYN_KW } ; [else] => { $ crate :: SyntaxKind :: ELSE_KW } ; [enum] => { $ crate :: SyntaxKind :: ENUM_KW } ; [extern] => { $ crate :: SyntaxKind :: EXTERN_KW } ; [false] => { $ crate :: SyntaxKind :: FALSE_KW } ; [fn] => { $ crate :: SyntaxKind :: FN_KW } ; [for] => { $ crate :: SyntaxKind :: FOR_KW } ; [if] => { $ crate :: SyntaxKind :: IF_KW } ; [impl] => { $ crate :: SyntaxKind :: IMPL_KW } ; [in] => { $ crate :: SyntaxKind :: IN_KW } ; [let] => { $ crate :: SyntaxKind :: LET_KW } ; [loop] => { $ crate :: SyntaxKind :: LOOP_KW } ; [macro] => { $ crate :: SyntaxKind :: MACRO_KW } ; [match] => { $ crate :: SyntaxKind :: MATCH_KW } ; [mod] => { $ crate :: SyntaxKind :: MOD_KW } ; [move] => { $ crate :: SyntaxKind :: MOVE_KW } ; [mut] => { $ crate :: SyntaxKind :: MUT_KW } ; [pub] => { $ crate :: SyntaxKind :: PUB_KW } ; [ref] => { $ crate :: SyntaxKind :: REF_KW } ; [return] => { $ crate :: SyntaxKind :: RETURN_KW } ; [self] => { $ crate :: SyntaxKind :: SELF_KW } ; [Self] => { $ crate :: SyntaxKind :: SELF_TYPE_KW } ; [static] => { $ crate :: SyntaxKind :: STATIC_KW } ; [struct] => { $ crate :: SyntaxKind :: STRUCT_KW } ; [super] => { $ crate :: SyntaxKind :: SUPER_KW } ; [trait] => { $ crate :: SyntaxKind :: TRAIT_KW } ; [true] => { $ crate :: SyntaxKind :: TRUE_KW } ; [try] => { $ crate :: SyntaxKind :: TRY_KW } ; [type] => { $ crate :: SyntaxKind :: TYPE_KW } ; [unsafe] => { $ crate :: SyntaxKind :: UNSAFE_KW } ; [use] => { $ crate :: SyntaxKind :: USE_KW } ; [where] => { $ crate :: SyntaxKind :: WHERE_KW } ; [while] => { $ crate :: SyntaxKind :: WHILE_KW } ; [yield] => { $ crate :: SyntaxKind :: YIELD_KW } ; [auto] => { $ crate :: SyntaxKind :: AUTO_KW } ; [default] => { $ crate :: SyntaxKind :: DEFAULT_KW } ; [existential] => { $ crate :: SyntaxKind :: EXISTENTIAL_KW } ; [union] => { $ crate :: SyntaxKind :: UNION_KW } ; [raw] => { $ crate :: SyntaxKind :: RAW_KW } ; [macro_rules] => { $ crate :: SyntaxKind :: MACRO_RULES_KW } ; [lifetime_ident] => { $ crate :: SyntaxKind :: LIFETIME_IDENT } ; [ident] => { $ crate :: SyntaxKind :: IDENT } ; [shebang] => { $ crate :: SyntaxKind :: SHEBANG } ; } pub use T; diff --git a/crates/parser/test_data/lexer/err/empty_exponent.rast b/crates/parser/test_data/lexer/err/empty_exponent.rast index 73de4cac24..af03d73ced 100644 --- a/crates/parser/test_data/lexer/err/empty_exponent.rast +++ b/crates/parser/test_data/lexer/err/empty_exponent.rast @@ -1,14 +1,14 @@ -FLOAT_NUMBER_START_0 "0e" error: Missing digits after the exponent symbol +FLOAT_NUMBER "0e" error: Missing digits after the exponent symbol WHITESPACE "\n" -FLOAT_NUMBER_START_0 "0E" error: Missing digits after the exponent symbol +FLOAT_NUMBER "0E" error: Missing digits after the exponent symbol WHITESPACE "\n\n" -FLOAT_NUMBER_START_0 "42e+" error: Missing digits after the exponent symbol +FLOAT_NUMBER "42e+" error: Missing digits after the exponent symbol WHITESPACE "\n" -FLOAT_NUMBER_START_0 "42e-" error: Missing digits after the exponent symbol +FLOAT_NUMBER "42e-" error: Missing digits after the exponent symbol WHITESPACE "\n" -FLOAT_NUMBER_START_0 "42E+" error: Missing digits after the exponent symbol +FLOAT_NUMBER "42E+" error: Missing digits after the exponent symbol WHITESPACE "\n" -FLOAT_NUMBER_START_0 "42E-" error: Missing digits after the exponent symbol +FLOAT_NUMBER "42E-" error: Missing digits after the exponent symbol WHITESPACE "\n\n" INT_NUMBER "42" DOT "." @@ -30,35 +30,19 @@ DOT "." IDENT "E" MINUS "-" WHITESPACE "\n\n" -FLOAT_NUMBER_START_2 "42" -DOT "." -FLOAT_NUMBER_PART "2e+" error: Missing digits after the exponent symbol +FLOAT_NUMBER "42.2e+" error: Missing digits after the exponent symbol WHITESPACE "\n" -FLOAT_NUMBER_START_2 "42" -DOT "." -FLOAT_NUMBER_PART "2e-" error: Missing digits after the exponent symbol +FLOAT_NUMBER "42.2e-" error: Missing digits after the exponent symbol WHITESPACE "\n" -FLOAT_NUMBER_START_2 "42" -DOT "." -FLOAT_NUMBER_PART "2E+" error: Missing digits after the exponent symbol +FLOAT_NUMBER "42.2E+" error: Missing digits after the exponent symbol WHITESPACE "\n" -FLOAT_NUMBER_START_2 "42" -DOT "." -FLOAT_NUMBER_PART "2E-" error: Missing digits after the exponent symbol +FLOAT_NUMBER "42.2E-" error: Missing digits after the exponent symbol WHITESPACE "\n\n" -FLOAT_NUMBER_START_2 "42" -DOT "." -FLOAT_NUMBER_PART "2e+f32" error: Missing digits after the exponent symbol +FLOAT_NUMBER "42.2e+f32" error: Missing digits after the exponent symbol WHITESPACE "\n" -FLOAT_NUMBER_START_2 "42" -DOT "." -FLOAT_NUMBER_PART "2e-f32" error: Missing digits after the exponent symbol +FLOAT_NUMBER "42.2e-f32" error: Missing digits after the exponent symbol WHITESPACE "\n" -FLOAT_NUMBER_START_2 "42" -DOT "." -FLOAT_NUMBER_PART "2E+f32" error: Missing digits after the exponent symbol +FLOAT_NUMBER "42.2E+f32" error: Missing digits after the exponent symbol WHITESPACE "\n" -FLOAT_NUMBER_START_2 "42" -DOT "." -FLOAT_NUMBER_PART "2E-f32" error: Missing digits after the exponent symbol +FLOAT_NUMBER "42.2E-f32" error: Missing digits after the exponent symbol WHITESPACE "\n" diff --git a/crates/parser/test_data/lexer/ok/numbers.rast b/crates/parser/test_data/lexer/ok/numbers.rast index 428bdf8a1f..8d13c3f610 100644 --- a/crates/parser/test_data/lexer/ok/numbers.rast +++ b/crates/parser/test_data/lexer/ok/numbers.rast @@ -4,8 +4,7 @@ INT_NUMBER "00" WHITESPACE " " INT_NUMBER "0_" WHITESPACE " " -FLOAT_NUMBER_START_1 "0" -DOT "." +FLOAT_NUMBER "0." WHITESPACE " " INT_NUMBER "0z" WHITESPACE "\n" @@ -21,13 +20,11 @@ INT_NUMBER "001279" WHITESPACE " " INT_NUMBER "0_1279" WHITESPACE " " -FLOAT_NUMBER_START_2 "0" -DOT "." -FLOAT_NUMBER_PART "1279" +FLOAT_NUMBER "0.1279" WHITESPACE " " -FLOAT_NUMBER_START_0 "0e1279" +FLOAT_NUMBER "0e1279" WHITESPACE " " -FLOAT_NUMBER_START_0 "0E1279" +FLOAT_NUMBER "0E1279" WHITESPACE "\n" INT_NUMBER "0" DOT "." @@ -40,7 +37,7 @@ IDENT "foo" L_PAREN "(" R_PAREN ")" WHITESPACE "\n" -FLOAT_NUMBER_START_0 "0e+1" +FLOAT_NUMBER "0e+1" WHITESPACE "\n" INT_NUMBER "0" DOT "." @@ -48,19 +45,13 @@ IDENT "e" PLUS "+" INT_NUMBER "1" WHITESPACE "\n" -FLOAT_NUMBER_START_2 "0" -DOT "." -FLOAT_NUMBER_PART "0E-2" +FLOAT_NUMBER "0.0E-2" WHITESPACE "\n" -FLOAT_NUMBER_START_2 "0___0" -DOT "." -FLOAT_NUMBER_PART "10000____0000e+111__" +FLOAT_NUMBER "0___0.10000____0000e+111__" WHITESPACE "\n" INT_NUMBER "1i64" WHITESPACE " " -FLOAT_NUMBER_START_2 "92" -DOT "." -FLOAT_NUMBER_PART "0f32" +FLOAT_NUMBER "92.0f32" WHITESPACE " " INT_NUMBER "11__s" WHITESPACE "\n" diff --git a/crates/parser/test_data/parser/err/0023_mismatched_paren.rast b/crates/parser/test_data/parser/err/0023_mismatched_paren.rast index 70cd030da1..4064a7a1ff 100644 --- a/crates/parser/test_data/parser/err/0023_mismatched_paren.rast +++ b/crates/parser/test_data/parser/err/0023_mismatched_paren.rast @@ -32,9 +32,7 @@ SOURCE_FILE INT_NUMBER "1" COMMA "," WHITESPACE " " - FLOAT_NUMBER_START_2 "2" - DOT "." - FLOAT_NUMBER_PART "0" + FLOAT_NUMBER "2.0" WHITESPACE "\n " R_CURLY "}" WHITESPACE " " diff --git a/crates/parser/test_data/parser/inline/ok/0011_field_expr.rast b/crates/parser/test_data/parser/inline/ok/0011_field_expr.rast index a1efb3a9fb..8498724b9e 100644 --- a/crates/parser/test_data/parser/inline/ok/0011_field_expr.rast +++ b/crates/parser/test_data/parser/inline/ok/0011_field_expr.rast @@ -41,39 +41,6 @@ SOURCE_FILE SEMICOLON ";" WHITESPACE "\n " EXPR_STMT - FIELD_EXPR - FIELD_EXPR - PATH_EXPR - PATH - PATH_SEGMENT - NAME_REF - IDENT "x" - DOT "." - NAME_REF - FLOAT_NUMBER_PART "0" - DOT "." - WHITESPACE " " - NAME_REF - IDENT "bar" - SEMICOLON ";" - WHITESPACE "\n " - EXPR_STMT - FIELD_EXPR - FIELD_EXPR - PATH_EXPR - PATH - PATH_SEGMENT - NAME_REF - IDENT "x" - DOT "." - NAME_REF - FLOAT_NUMBER_PART "0" - DOT "." - NAME_REF - FLOAT_NUMBER_PART "1" - SEMICOLON ";" - WHITESPACE "\n " - EXPR_STMT CALL_EXPR FIELD_EXPR PATH_EXPR diff --git a/crates/parser/test_data/parser/inline/ok/0011_field_expr.rs b/crates/parser/test_data/parser/inline/ok/0011_field_expr.rs index 551b1ecaf0..b8da2ddc30 100644 --- a/crates/parser/test_data/parser/inline/ok/0011_field_expr.rs +++ b/crates/parser/test_data/parser/inline/ok/0011_field_expr.rs @@ -1,7 +1,5 @@ fn foo() { x.foo; x.0.bar; - x.0. bar; - x.0.1; x.0(); } diff --git a/crates/parser/test_data/parser/inline/ok/0085_expr_literals.rast b/crates/parser/test_data/parser/inline/ok/0085_expr_literals.rast index b3236976b9..403c265ea3 100644 --- a/crates/parser/test_data/parser/inline/ok/0085_expr_literals.rast +++ b/crates/parser/test_data/parser/inline/ok/0085_expr_literals.rast @@ -57,10 +57,7 @@ SOURCE_FILE EQ "=" WHITESPACE " " LITERAL - FLOAT_LITERAL - FLOAT_NUMBER_START_2 "2" - DOT "." - FLOAT_NUMBER_PART "0" + FLOAT_NUMBER "2.0" SEMICOLON ";" WHITESPACE "\n " LET_STMT diff --git a/crates/parser/test_data/parser/inline/ok/0107_method_call_expr.rast b/crates/parser/test_data/parser/inline/ok/0107_method_call_expr.rast index 69f1055b7e..dcbcfe1231 100644 --- a/crates/parser/test_data/parser/inline/ok/0107_method_call_expr.rast +++ b/crates/parser/test_data/parser/inline/ok/0107_method_call_expr.rast @@ -58,32 +58,6 @@ SOURCE_FILE COMMA "," R_PAREN ")" SEMICOLON ";" - WHITESPACE "\n\n " - EXPR_STMT - METHOD_CALL_EXPR - LITERAL - FLOAT_LITERAL - FLOAT_NUMBER_START_0 "0e0" - DOT "." - NAME_REF - IDENT "sin" - ARG_LIST - L_PAREN "(" - R_PAREN ")" - SEMICOLON ";" - WHITESPACE "\n " - EXPR_STMT - METHOD_CALL_EXPR - LITERAL - FLOAT_LITERAL - FLOAT_NUMBER_START_0 "0e0f32" - DOT "." - NAME_REF - IDENT "sin" - ARG_LIST - L_PAREN "(" - R_PAREN ")" - SEMICOLON ";" WHITESPACE "\n" R_CURLY "}" WHITESPACE "\n" diff --git a/crates/parser/test_data/parser/inline/ok/0107_method_call_expr.rs b/crates/parser/test_data/parser/inline/ok/0107_method_call_expr.rs index 3e5d464e23..1a3aa35ae8 100644 --- a/crates/parser/test_data/parser/inline/ok/0107_method_call_expr.rs +++ b/crates/parser/test_data/parser/inline/ok/0107_method_call_expr.rs @@ -1,7 +1,4 @@ fn foo() { x.foo(); y.bar::<T>(1, 2,); - - 0e0.sin(); - 0e0f32.sin(); } diff --git a/crates/parser/test_data/parser/inline/ok/0201_float_literal.rast b/crates/parser/test_data/parser/inline/ok/0201_float_literal.rast deleted file mode 100644 index df4fb6eb41..0000000000 --- a/crates/parser/test_data/parser/inline/ok/0201_float_literal.rast +++ /dev/null @@ -1,51 +0,0 @@ -SOURCE_FILE - FN - FN_KW "fn" - WHITESPACE " " - NAME - IDENT "f" - PARAM_LIST - L_PAREN "(" - R_PAREN ")" - WHITESPACE " " - BLOCK_EXPR - STMT_LIST - L_CURLY "{" - WHITESPACE "\n " - EXPR_STMT - LITERAL - FLOAT_LITERAL - FLOAT_NUMBER_START_2 "0" - DOT "." - FLOAT_NUMBER_PART "0" - SEMICOLON ";" - WHITESPACE "\n " - EXPR_STMT - LITERAL - FLOAT_LITERAL - FLOAT_NUMBER_START_1 "1" - DOT "." - SEMICOLON ";" - WHITESPACE "\n " - EXPR_STMT - LITERAL - FLOAT_LITERAL - FLOAT_NUMBER_START_0 "0e0" - SEMICOLON ";" - WHITESPACE "\n " - EXPR_STMT - LITERAL - FLOAT_LITERAL - FLOAT_NUMBER_START_0 "0e0f32" - SEMICOLON ";" - WHITESPACE "\n " - EXPR_STMT - LITERAL - FLOAT_LITERAL - FLOAT_NUMBER_START_2 "1" - DOT "." - FLOAT_NUMBER_PART "23f64" - SEMICOLON ";" - WHITESPACE "\n" - R_CURLY "}" - WHITESPACE "\n" diff --git a/crates/parser/test_data/parser/inline/ok/0201_float_literal.rs b/crates/parser/test_data/parser/inline/ok/0201_float_literal.rs deleted file mode 100644 index 0d51ec1252..0000000000 --- a/crates/parser/test_data/parser/inline/ok/0201_float_literal.rs +++ /dev/null @@ -1,7 +0,0 @@ -fn f() { - 0.0; - 1.; - 0e0; - 0e0f32; - 1.23f64; -} diff --git a/crates/parser/test_data/parser/ok/0056_neq_in_type.rast b/crates/parser/test_data/parser/ok/0056_neq_in_type.rast index b2f66e2f1e..55ce31275f 100644 --- a/crates/parser/test_data/parser/ok/0056_neq_in_type.rast +++ b/crates/parser/test_data/parser/ok/0056_neq_in_type.rast @@ -19,10 +19,7 @@ SOURCE_FILE CAST_EXPR METHOD_CALL_EXPR LITERAL - FLOAT_LITERAL - FLOAT_NUMBER_START_2 "1" - DOT "." - FLOAT_NUMBER_PART "0f32" + FLOAT_NUMBER "1.0f32" DOT "." NAME_REF IDENT "floor" @@ -43,10 +40,7 @@ SOURCE_FILE CAST_EXPR METHOD_CALL_EXPR LITERAL - FLOAT_LITERAL - FLOAT_NUMBER_START_2 "1" - DOT "." - FLOAT_NUMBER_PART "0f32" + FLOAT_NUMBER "1.0f32" DOT "." NAME_REF IDENT "floor" diff --git a/crates/syntax/rust.ungram b/crates/syntax/rust.ungram index 85270dee62..62aa478399 100644 --- a/crates/syntax/rust.ungram +++ b/crates/syntax/rust.ungram @@ -365,20 +365,13 @@ MacroExpr = Literal = Attr* value:( - 'int_number' | FloatLiteral + 'int_number' | 'float_number' | 'string' | 'raw_string' | 'byte_string' | 'raw_byte_string' | 'true' | 'false' | 'char' | 'byte' ) -FloatLiteral = - 'float_number_start_0'? - 'float_number_start_1'? - 'float_number_start_2'? - '.'? - 'float_number_part'? - PathExpr = Attr* Path diff --git a/crates/syntax/src/ast/expr_ext.rs b/crates/syntax/src/ast/expr_ext.rs index 11d81d4778..17785152bc 100644 --- a/crates/syntax/src/ast/expr_ext.rs +++ b/crates/syntax/src/ast/expr_ext.rs @@ -8,7 +8,7 @@ use crate::{ operators::{ArithOp, BinaryOp, CmpOp, LogicOp, Ordering, RangeOp, UnaryOp}, support, AstChildren, AstNode, }, - AstToken, SyntaxElement, + AstToken, SyntaxKind::*, SyntaxNode, SyntaxToken, T, }; @@ -282,32 +282,30 @@ pub enum LiteralKind { String(ast::String), ByteString(ast::ByteString), IntNumber(ast::IntNumber), - FloatNumber(ast::FloatLiteral), + FloatNumber(ast::FloatNumber), Char(ast::Char), Byte(ast::Byte), Bool(bool), } impl ast::Literal { - pub fn value(&self) -> SyntaxElement { + pub fn token(&self) -> SyntaxToken { self.syntax() .children_with_tokens() .find(|e| e.kind() != ATTR && !e.kind().is_trivia()) + .and_then(|e| e.into_token()) .unwrap() } + pub fn kind(&self) -> LiteralKind { - let token = match self.value() { - rowan::NodeOrToken::Node(node) => { - return LiteralKind::FloatNumber( - ast::FloatLiteral::cast(node).expect("unreachable"), - ); - } - rowan::NodeOrToken::Token(token) => token, - }; + let token = self.token(); if let Some(t) = ast::IntNumber::cast(token.clone()) { return LiteralKind::IntNumber(t); } + if let Some(t) = ast::FloatNumber::cast(token.clone()) { + return LiteralKind::FloatNumber(t); + } if let Some(t) = ast::String::cast(token.clone()) { return LiteralKind::String(t); } @@ -327,26 +325,6 @@ impl ast::Literal { _ => unreachable!(), } } - - pub fn as_string(&self) -> Option<ast::String> { - match self.kind() { - LiteralKind::String(it) => Some(it), - _ => None, - } - } - - pub fn as_byte_string(&self) -> Option<ast::ByteString> { - match self.kind() { - LiteralKind::ByteString(it) => Some(it), - _ => None, - } - } -} - -impl ast::FloatLiteral { - pub fn suffix(&self) -> Option<String> { - ast::FloatNumberPart::cast(self.syntax().last_token()?)?.suffix().map(|s| s.to_string()) - } } pub enum BlockModifier { @@ -386,7 +364,7 @@ impl ast::BlockExpr { fn test_literal_with_attr() { let parse = ast::SourceFile::parse(r#"const _: &str = { #[attr] "Hello" };"#); let lit = parse.tree().syntax().descendants().find_map(ast::Literal::cast).unwrap(); - assert_eq!(lit.value().to_string(), r#""Hello""#); + assert_eq!(lit.token().text(), r#""Hello""#); } impl ast::RecordExprField { diff --git a/crates/syntax/src/ast/generated/nodes.rs b/crates/syntax/src/ast/generated/nodes.rs index 9d5af8e63c..cf90ba64cf 100644 --- a/crates/syntax/src/ast/generated/nodes.rs +++ b/crates/syntax/src/ast/generated/nodes.rs @@ -1086,26 +1086,6 @@ impl UnderscoreExpr { } #[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub struct FloatLiteral { - pub(crate) syntax: SyntaxNode, -} -impl FloatLiteral { - pub fn float_number_start_0_token(&self) -> Option<SyntaxToken> { - support::token(&self.syntax, T![float_number_start_0]) - } - pub fn float_number_start_1_token(&self) -> Option<SyntaxToken> { - support::token(&self.syntax, T![float_number_start_1]) - } - pub fn float_number_start_2_token(&self) -> Option<SyntaxToken> { - support::token(&self.syntax, T![float_number_start_2]) - } - pub fn dot_token(&self) -> Option<SyntaxToken> { support::token(&self.syntax, T![.]) } - pub fn float_number_part_token(&self) -> Option<SyntaxToken> { - support::token(&self.syntax, T![float_number_part]) - } -} - -#[derive(Debug, Clone, PartialEq, Eq, Hash)] pub struct StmtList { pub(crate) syntax: SyntaxNode, } @@ -2739,17 +2719,6 @@ impl AstNode for UnderscoreExpr { } fn syntax(&self) -> &SyntaxNode { &self.syntax } } -impl AstNode for FloatLiteral { - fn can_cast(kind: SyntaxKind) -> bool { kind == FLOAT_LITERAL } - fn cast(syntax: SyntaxNode) -> Option<Self> { - if Self::can_cast(syntax.kind()) { - Some(Self { syntax }) - } else { - None - } - } - fn syntax(&self) -> &SyntaxNode { &self.syntax } -} impl AstNode for StmtList { fn can_cast(kind: SyntaxKind) -> bool { kind == STMT_LIST } fn cast(syntax: SyntaxNode) -> Option<Self> { @@ -4639,11 +4608,6 @@ impl std::fmt::Display for UnderscoreExpr { std::fmt::Display::fmt(self.syntax(), f) } } -impl std::fmt::Display for FloatLiteral { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - std::fmt::Display::fmt(self.syntax(), f) - } -} impl std::fmt::Display for StmtList { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { std::fmt::Display::fmt(self.syntax(), f) diff --git a/crates/syntax/src/ast/generated/tokens.rs b/crates/syntax/src/ast/generated/tokens.rs index 1e1a55e326..a3209c5abd 100644 --- a/crates/syntax/src/ast/generated/tokens.rs +++ b/crates/syntax/src/ast/generated/tokens.rs @@ -112,16 +112,16 @@ impl AstToken for IntNumber { } #[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub struct FloatNumberPart { +pub struct FloatNumber { pub(crate) syntax: SyntaxToken, } -impl std::fmt::Display for FloatNumberPart { +impl std::fmt::Display for FloatNumber { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { std::fmt::Display::fmt(&self.syntax, f) } } -impl AstToken for FloatNumberPart { - fn can_cast(kind: SyntaxKind) -> bool { kind == FLOAT_NUMBER_PART } +impl AstToken for FloatNumber { + fn can_cast(kind: SyntaxKind) -> bool { kind == FLOAT_NUMBER } fn cast(syntax: SyntaxToken) -> Option<Self> { if Self::can_cast(syntax.kind()) { Some(Self { syntax }) diff --git a/crates/syntax/src/ast/make.rs b/crates/syntax/src/ast/make.rs index 846f4f3c71..5908dda8e6 100644 --- a/crates/syntax/src/ast/make.rs +++ b/crates/syntax/src/ast/make.rs @@ -799,11 +799,6 @@ pub fn struct_( )) } -pub fn literal(text: &str) -> ast::Literal { - assert_eq!(text.trim(), text); - ast_from_text(&format!("fn f() {{ let _ = {}; }}", text)) -} - #[track_caller] fn ast_from_text<N: AstNode>(text: &str) -> N { let parse = SourceFile::parse(text); @@ -832,7 +827,7 @@ pub fn token(kind: SyntaxKind) -> SyntaxToken { pub mod tokens { use once_cell::sync::Lazy; - use crate::{AstNode, Parse, SourceFile, SyntaxKind::*, SyntaxToken}; + use crate::{ast, AstNode, Parse, SourceFile, SyntaxKind::*, SyntaxToken}; pub(super) static SOURCE_FILE: Lazy<Parse<SourceFile>> = Lazy::new(|| { SourceFile::parse( @@ -863,6 +858,12 @@ pub mod tokens { sf.syntax().first_child_or_token().unwrap().into_token().unwrap() } + pub fn literal(text: &str) -> SyntaxToken { + assert_eq!(text.trim(), text); + let lit: ast::Literal = super::ast_from_text(&format!("fn f() {{ let _ = {}; }}", text)); + lit.syntax().first_child_or_token().unwrap().into_token().unwrap() + } + pub fn single_newline() -> SyntaxToken { let res = SOURCE_FILE .tree() diff --git a/crates/syntax/src/ast/node_ext.rs b/crates/syntax/src/ast/node_ext.rs index 5b19b5ed2c..f2153ca921 100644 --- a/crates/syntax/src/ast/node_ext.rs +++ b/crates/syntax/src/ast/node_ext.rs @@ -555,9 +555,7 @@ impl ast::FieldExpr { self.syntax .children_with_tokens() // FIXME: Accepting floats here to reject them in validation later - .find(|c| { - c.kind() == SyntaxKind::INT_NUMBER || c.kind() == SyntaxKind::FLOAT_NUMBER_PART - }) + .find(|c| c.kind() == SyntaxKind::INT_NUMBER || c.kind() == SyntaxKind::FLOAT_NUMBER) .as_ref() .and_then(SyntaxElement::as_token) .cloned() diff --git a/crates/syntax/src/ast/token_ext.rs b/crates/syntax/src/ast/token_ext.rs index 5f2e7231d9..4b6dc236b5 100644 --- a/crates/syntax/src/ast/token_ext.rs +++ b/crates/syntax/src/ast/token_ext.rs @@ -321,7 +321,7 @@ impl ast::IntNumber { } } -impl ast::FloatNumberPart { +impl ast::FloatNumber { pub fn suffix(&self) -> Option<&str> { let text = self.text(); let mut indices = text.char_indices(); @@ -355,24 +355,14 @@ impl Radix { #[cfg(test)] mod tests { - use crate::ast::{self, make}; + use crate::ast::{self, make, FloatNumber, IntNumber}; fn check_float_suffix<'a>(lit: &str, expected: impl Into<Option<&'a str>>) { - let suffix = match make::literal(lit).kind() { - ast::LiteralKind::FloatNumber(f) => f.suffix(), - // `1f32` lexes as an INT_NUMBER - ast::LiteralKind::IntNumber(i) => i.suffix().map(|s| s.to_string()), - e => unreachable!("{e:?}"), - }; - assert_eq!(suffix.as_deref(), expected.into()); + assert_eq!(FloatNumber { syntax: make::tokens::literal(lit) }.suffix(), expected.into()); } fn check_int_suffix<'a>(lit: &str, expected: impl Into<Option<&'a str>>) { - let i = match make::literal(lit).kind() { - ast::LiteralKind::IntNumber(i) => i, - _ => unreachable!(), - }; - assert_eq!(i.suffix(), expected.into()); + assert_eq!(IntNumber { syntax: make::tokens::literal(lit) }.suffix(), expected.into()); } #[test] @@ -400,11 +390,12 @@ mod tests { } fn check_string_value<'a>(lit: &str, expected: impl Into<Option<&'a str>>) { - let s = match make::literal(&format!("\"{}\"", lit)).kind() { - ast::LiteralKind::String(s) => s, - _ => unreachable!(), - }; - assert_eq!(s.value().as_deref(), expected.into()); + assert_eq!( + ast::String { syntax: make::tokens::literal(&format!("\"{}\"", lit)) } + .value() + .as_deref(), + expected.into() + ); } #[test] diff --git a/crates/syntax/src/tests/ast_src.rs b/crates/syntax/src/tests/ast_src.rs index f5a78e4119..2f6932a1ad 100644 --- a/crates/syntax/src/tests/ast_src.rs +++ b/crates/syntax/src/tests/ast_src.rs @@ -71,17 +71,7 @@ pub(crate) const KINDS_SRC: KindsSrc = KindsSrc { "super", "trait", "true", "try", "type", "unsafe", "use", "where", "while", "yield", ], contextual_keywords: &["auto", "default", "existential", "union", "raw", "macro_rules"], - literals: &[ - "INT_NUMBER", - "FLOAT_NUMBER_START_0", - "FLOAT_NUMBER_START_1", - "FLOAT_NUMBER_START_2", - "FLOAT_NUMBER_PART", - "CHAR", - "BYTE", - "STRING", - "BYTE_STRING", - ], + literals: &["INT_NUMBER", "FLOAT_NUMBER", "CHAR", "BYTE", "STRING", "BYTE_STRING"], tokens: &["ERROR", "IDENT", "WHITESPACE", "LIFETIME_IDENT", "COMMENT", "SHEBANG"], nodes: &[ "SOURCE_FILE", @@ -193,7 +183,6 @@ pub(crate) const KINDS_SRC: KindsSrc = KindsSrc { "PATH", "PATH_SEGMENT", "LITERAL", - "FLOAT_LITERAL", "RENAME", "VISIBILITY", "WHERE_CLAUSE", diff --git a/crates/syntax/src/tests/sourcegen_ast.rs b/crates/syntax/src/tests/sourcegen_ast.rs index 33bf2c0cba..4cfb8075cb 100644 --- a/crates/syntax/src/tests/sourcegen_ast.rs +++ b/crates/syntax/src/tests/sourcegen_ast.rs @@ -462,10 +462,6 @@ fn generate_syntax_kinds(grammar: KindsSrc<'_>) -> String { [lifetime_ident] => { $crate::SyntaxKind::LIFETIME_IDENT }; [ident] => { $crate::SyntaxKind::IDENT }; [shebang] => { $crate::SyntaxKind::SHEBANG }; - [float_number_part] => { $crate::SyntaxKind::FLOAT_NUMBER_PART }; - [float_number_start_0] => { $crate::SyntaxKind::FLOAT_NUMBER_START_0 }; - [float_number_start_1] => { $crate::SyntaxKind::FLOAT_NUMBER_START_1 }; - [float_number_start_2] => { $crate::SyntaxKind::FLOAT_NUMBER_START_2 }; } pub use T; }; @@ -589,7 +585,7 @@ impl Field { fn lower(grammar: &Grammar) -> AstSrc { let mut res = AstSrc { - tokens: "Whitespace Comment String ByteString IntNumber FloatNumberPart Char Byte Ident" + tokens: "Whitespace Comment String ByteString IntNumber FloatNumber Char Byte Ident" .split_ascii_whitespace() .map(|it| it.to_string()) .collect::<Vec<_>>(), diff --git a/crates/syntax/src/validation.rs b/crates/syntax/src/validation.rs index 3edca3eb8f..c2c2c82e11 100644 --- a/crates/syntax/src/validation.rs +++ b/crates/syntax/src/validation.rs @@ -119,15 +119,8 @@ fn validate_literal(literal: ast::Literal, acc: &mut Vec<SyntaxError>) { text.rfind(end_delimiter).and_then(|end| text.get(prefix_len..end)) } - let token = literal.value(); - let text; - let text = match &token { - rowan::NodeOrToken::Node(node) => { - text = node.text().to_string(); - &*text - } - rowan::NodeOrToken::Token(token) => token.text(), - }; + let token = literal.token(); + let text = token.text(); // FIXME: lift this lambda refactor to `fn` (https://github.com/rust-analyzer/rust-analyzer/pull/2834#discussion_r366199205) let mut push_err = |prefix_len, (off, err): (usize, unescape::EscapeError)| { |