Unnamed repository; edit this file 'description' to name the repository.
Recover better for more delimited sequences
20 files changed, 153 insertions, 124 deletions
diff --git a/crates/hir-def/src/macro_expansion_tests/mbe/regression.rs b/crates/hir-def/src/macro_expansion_tests/mbe/regression.rs index 910917ac68..8358a46f0a 100644 --- a/crates/hir-def/src/macro_expansion_tests/mbe/regression.rs +++ b/crates/hir-def/src/macro_expansion_tests/mbe/regression.rs @@ -830,7 +830,6 @@ macro_rules! rgb_color { /* parse error: expected COMMA */ /* parse error: expected R_ANGLE */ /* parse error: expected SEMICOLON */ -/* parse error: expected SEMICOLON */ /* parse error: expected expression, item or let statement */ pub fn new() { let _ = 0as u32<<(8+8); @@ -848,21 +847,21 @@ pub fn new() { // [email protected] "{" // [email protected] "let" // [email protected] "_" // [email protected] "=" // [email protected] "0" // [email protected] "as" // [email protected] "u32" // [email protected] "<" @@ -877,9 +876,9 @@ pub fn new() { // [email protected] "8" // [email protected] "+" -// [email protected] "8" +// [email protected] "8" // [email protected] ")" // [email protected] ";" diff --git a/crates/ide/src/syntax_highlighting/tests.rs b/crates/ide/src/syntax_highlighting/tests.rs index 2f870d769c..fc9b5d3ba4 100644 --- a/crates/ide/src/syntax_highlighting/tests.rs +++ b/crates/ide/src/syntax_highlighting/tests.rs @@ -1126,5 +1126,5 @@ fn benchmark_syntax_highlighting_parser() { .filter(|it| it.highlight.tag == HlTag::Symbol(SymbolKind::Function)) .count() }; - assert_eq!(hash, 1609); + assert_eq!(hash, 1608); } diff --git a/crates/parser/src/grammar.rs b/crates/parser/src/grammar.rs index 485b612f08..15ec9e167e 100644 --- a/crates/parser/src/grammar.rs +++ b/crates/parser/src/grammar.rs @@ -200,6 +200,8 @@ impl BlockLike { } } +const VISIBILITY_FIRST: TokenSet = TokenSet::new(&[T![pub], T![crate]]); + fn opt_visibility(p: &mut Parser<'_>, in_tuple_field: bool) -> bool { match p.current() { T![pub] => { @@ -340,3 +342,31 @@ fn error_block(p: &mut Parser<'_>, message: &str) { p.eat(T!['}']); m.complete(p, ERROR); } + +/// The `parser` passed this is required to at least consume one token if it returns `true`. +/// If the `parser` returns false, parsing will stop. +fn delimited( + p: &mut Parser<'_>, + bra: SyntaxKind, + ket: SyntaxKind, + delim: SyntaxKind, + first_set: TokenSet, + mut parser: impl FnMut(&mut Parser<'_>) -> bool, +) { + p.bump(bra); + while !p.at(ket) && !p.at(EOF) { + if !parser(p) { + break; + } + if !p.at(delim) { + if p.at_ts(first_set) { + p.error(format!("expected {:?}", delim)); + } else { + break; + } + } else { + p.bump(delim); + } + } + p.expect(ket); +} diff --git a/crates/parser/src/grammar/attributes.rs b/crates/parser/src/grammar/attributes.rs index 0cf6a16f86..4ecaa6e6a8 100644 --- a/crates/parser/src/grammar/attributes.rs +++ b/crates/parser/src/grammar/attributes.rs @@ -1,5 +1,7 @@ use super::*; +pub(super) const ATTRIBUTE_FIRST: TokenSet = TokenSet::new(&[T![#]]); + pub(super) fn inner_attrs(p: &mut Parser<'_>) { while p.at(T![#]) && p.nth(1) == T![!] { attr(p, true); diff --git a/crates/parser/src/grammar/expressions.rs b/crates/parser/src/grammar/expressions.rs index 4d6097b5ab..4b080102a2 100644 --- a/crates/parser/src/grammar/expressions.rs +++ b/crates/parser/src/grammar/expressions.rs @@ -1,5 +1,7 @@ mod atom; +use crate::grammar::attributes::ATTRIBUTE_FIRST; + use super::*; pub(crate) use self::atom::{block_expr, match_arm_list}; @@ -572,27 +574,11 @@ fn cast_expr(p: &mut Parser<'_>, lhs: CompletedMarker) -> CompletedMarker { fn arg_list(p: &mut Parser<'_>) { assert!(p.at(T!['('])); let m = p.start(); - p.bump(T!['(']); - while !p.at(T![')']) && !p.at(EOF) { - // test arg_with_attr - // fn main() { - // foo(#[attr] 92) - // } - if !expr(p) { - break; - } - if !p.at(T![,]) { - if p.at_ts(EXPR_FIRST) { - p.error("expected `,`"); - continue; - } else { - break; - } - } else { - p.bump(T![,]); - } - } - p.eat(T![')']); + // test arg_with_attr + // fn main() { + // foo(#[attr] 92) + // } + delimited(p, T!['('], T![')'], T![,], EXPR_FIRST.union(ATTRIBUTE_FIRST), expr); m.complete(p, ARG_LIST); } diff --git a/crates/parser/src/grammar/expressions/atom.rs b/crates/parser/src/grammar/expressions/atom.rs index c3c5d474cb..efc2603835 100644 --- a/crates/parser/src/grammar/expressions/atom.rs +++ b/crates/parser/src/grammar/expressions/atom.rs @@ -118,7 +118,7 @@ pub(super) fn atom_expr( // fn main() { // 'loop: impl // } - p.error("expected a loop"); + p.error("expected a loop or block"); m.complete(p, ERROR); return None; } diff --git a/crates/parser/src/grammar/generic_args.rs b/crates/parser/src/grammar/generic_args.rs index c438943a00..919d9b91eb 100644 --- a/crates/parser/src/grammar/generic_args.rs +++ b/crates/parser/src/grammar/generic_args.rs @@ -5,27 +5,35 @@ pub(super) fn opt_generic_arg_list(p: &mut Parser<'_>, colon_colon_required: boo if p.at(T![::]) && p.nth(2) == T![<] { m = p.start(); p.bump(T![::]); - p.bump(T![<]); } else if !colon_colon_required && p.at(T![<]) && p.nth(1) != T![=] { m = p.start(); - p.bump(T![<]); } else { return; } - while !p.at(EOF) && !p.at(T![>]) { - generic_arg(p); - if !p.at(T![>]) && !p.expect(T![,]) { - break; - } - } - p.expect(T![>]); + delimited(p, T![<], T![>], T![,], GENERIC_ARG_FIRST, generic_arg); m.complete(p, GENERIC_ARG_LIST); } +const GENERIC_ARG_FIRST: TokenSet = TokenSet::new(&[ + LIFETIME_IDENT, + IDENT, + T!['{'], + T![true], + T![false], + T![-], + INT_NUMBER, + FLOAT_NUMBER, + CHAR, + BYTE, + STRING, + BYTE_STRING, +]) +.union(types::TYPE_FIRST); + // test generic_arg // type T = S<i32>; -fn generic_arg(p: &mut Parser<'_>) { +fn generic_arg(p: &mut Parser<'_>) -> bool { match p.current() { LIFETIME_IDENT => lifetime_arg(p), T!['{'] | T![true] | T![false] | T![-] => const_arg(p), @@ -68,8 +76,10 @@ fn generic_arg(p: &mut Parser<'_>) { } } } - _ => type_arg(p), + _ if p.at_ts(types::TYPE_FIRST) => type_arg(p), + _ => return false, } + true } // test lifetime_arg diff --git a/crates/parser/src/grammar/generic_params.rs b/crates/parser/src/grammar/generic_params.rs index 6db28ef132..7fcf938bab 100644 --- a/crates/parser/src/grammar/generic_params.rs +++ b/crates/parser/src/grammar/generic_params.rs @@ -1,3 +1,5 @@ +use crate::grammar::attributes::ATTRIBUTE_FIRST; + use super::*; pub(super) fn opt_generic_param_list(p: &mut Parser<'_>) { @@ -11,32 +13,31 @@ pub(super) fn opt_generic_param_list(p: &mut Parser<'_>) { fn generic_param_list(p: &mut Parser<'_>) { assert!(p.at(T![<])); let m = p.start(); - p.bump(T![<]); + delimited(p, T![<], T![>], T![,], GENERIC_PARAM_FIRST.union(ATTRIBUTE_FIRST), |p| { + // test generic_param_attribute + // fn foo<#[lt_attr] 'a, #[t_attr] T>() {} + let m = p.start(); + attributes::outer_attrs(p); + generic_param(p, m) + }); - while !p.at(EOF) && !p.at(T![>]) { - generic_param(p); - if !p.at(T![>]) && !p.expect(T![,]) { - break; - } - } - p.expect(T![>]); m.complete(p, GENERIC_PARAM_LIST); } -fn generic_param(p: &mut Parser<'_>) { - let m = p.start(); - // test generic_param_attribute - // fn foo<#[lt_attr] 'a, #[t_attr] T>() {} - attributes::outer_attrs(p); +const GENERIC_PARAM_FIRST: TokenSet = TokenSet::new(&[IDENT, LIFETIME_IDENT, T![const]]); + +fn generic_param(p: &mut Parser<'_>, m: Marker) -> bool { match p.current() { LIFETIME_IDENT => lifetime_param(p, m), IDENT => type_param(p, m), T![const] => const_param(p, m), _ => { m.abandon(p); - p.err_and_bump("expected type parameter"); + p.err_and_bump("expected generic parameter"); + return false; } } + true } // test lifetime_param diff --git a/crates/parser/src/grammar/items/adt.rs b/crates/parser/src/grammar/items/adt.rs index e7d30516b9..17f41b8e13 100644 --- a/crates/parser/src/grammar/items/adt.rs +++ b/crates/parser/src/grammar/items/adt.rs @@ -1,3 +1,5 @@ +use crate::grammar::attributes::ATTRIBUTE_FIRST; + use super::*; // test struct_item @@ -141,28 +143,31 @@ pub(crate) fn record_field_list(p: &mut Parser<'_>) { } } +const TUPLE_FIELD_FIRST: TokenSet = + types::TYPE_FIRST.union(ATTRIBUTE_FIRST).union(VISIBILITY_FIRST); + fn tuple_field_list(p: &mut Parser<'_>) { assert!(p.at(T!['('])); let m = p.start(); - p.bump(T!['(']); - while !p.at(T![')']) && !p.at(EOF) { + delimited(p, T!['('], T![')'], T![,], TUPLE_FIELD_FIRST, |p| { let m = p.start(); // test tuple_field_attrs // struct S (#[attr] f32); attributes::outer_attrs(p); - opt_visibility(p, true); + let has_vis = opt_visibility(p, true); if !p.at_ts(types::TYPE_FIRST) { p.error("expected a type"); - m.complete(p, ERROR); - break; + if has_vis { + m.complete(p, ERROR); + } else { + m.abandon(p); + } + return false; } types::type_(p); m.complete(p, TUPLE_FIELD); + true + }); - if !p.at(T![')']) { - p.expect(T![,]); - } - } - p.expect(T![')']); m.complete(p, TUPLE_FIELD_LIST); } diff --git a/crates/parser/src/grammar/params.rs b/crates/parser/src/grammar/params.rs index 20e8e95f06..74eae9151a 100644 --- a/crates/parser/src/grammar/params.rs +++ b/crates/parser/src/grammar/params.rs @@ -1,3 +1,5 @@ +use crate::grammar::attributes::ATTRIBUTE_FIRST; + use super::*; // test param_list @@ -66,14 +68,20 @@ fn list_(p: &mut Parser<'_>, flavor: Flavor) { } }; - if !p.at_ts(PARAM_FIRST) { + if !p.at_ts(PARAM_FIRST.union(ATTRIBUTE_FIRST)) { p.error("expected value parameter"); m.abandon(p); break; } param(p, m, flavor); - if !p.at(ket) { - p.expect(T![,]); + if !p.at(T![,]) { + if p.at_ts(PARAM_FIRST.union(ATTRIBUTE_FIRST)) { + p.error("expected `,`"); + } else { + break; + } + } else { + p.bump(T![,]); } } diff --git a/crates/parser/test_data/parser/err/0009_broken_struct_type_parameter.rast b/crates/parser/test_data/parser/err/0009_broken_struct_type_parameter.rast index a015432171..cdc01863ab 100644 --- a/crates/parser/test_data/parser/err/0009_broken_struct_type_parameter.rast +++ b/crates/parser/test_data/parser/err/0009_broken_struct_type_parameter.rast @@ -44,8 +44,7 @@ SOURCE_FILE IDENT "T" SEMICOLON ";" WHITESPACE "\n" -error 9: expected type parameter -error 11: expected COMMA +error 9: expected generic parameter error 11: expected R_ANGLE error 11: expected `;`, `{`, or `(` error 12: expected an item diff --git a/crates/parser/test_data/parser/err/0013_invalid_type.rast b/crates/parser/test_data/parser/err/0013_invalid_type.rast index eec84a0c67..b485c71ab3 100644 --- a/crates/parser/test_data/parser/err/0013_invalid_type.rast +++ b/crates/parser/test_data/parser/err/0013_invalid_type.rast @@ -43,17 +43,14 @@ SOURCE_FILE IDENT "Box" GENERIC_ARG_LIST L_ANGLE "<" - TYPE_ARG - ERROR - AT "@" - WHITESPACE " " - TUPLE_FIELD - PATH_TYPE - PATH - PATH_SEGMENT - NAME_REF - IDENT "Any" - ERROR + ERROR + AT "@" + WHITESPACE " " + MACRO_CALL + PATH + PATH_SEGMENT + NAME_REF + IDENT "Any" ERROR R_ANGLE ">" ERROR @@ -69,17 +66,14 @@ SOURCE_FILE ERROR SEMICOLON ";" WHITESPACE "\n\n" -error 67: expected type -error 68: expected COMMA -error 68: expected R_ANGLE -error 68: expected COMMA -error 68: expected R_ANGLE -error 68: expected COMMA -error 68: expected R_ANGLE -error 68: expected COMMA -error 72: expected COMMA -error 72: expected a type -error 72: expected R_PAREN +error 67: expected R_ANGLE +error 67: expected R_ANGLE +error 67: expected R_ANGLE +error 67: expected R_PAREN +error 67: expected SEMICOLON +error 67: expected an item +error 72: expected BANG +error 72: expected `{`, `[`, `(` error 72: expected SEMICOLON error 72: expected an item error 73: expected an item diff --git a/crates/parser/test_data/parser/err/0022_bad_exprs.rast b/crates/parser/test_data/parser/err/0022_bad_exprs.rast index 36a025dc0a..d97fc6c720 100644 --- a/crates/parser/test_data/parser/err/0022_bad_exprs.rast +++ b/crates/parser/test_data/parser/err/0022_bad_exprs.rast @@ -151,6 +151,7 @@ error 26: expected `;`, `{`, or `(` error 30: expected pattern error 31: expected SEMICOLON error 53: expected expression +error 54: expected R_PAREN error 54: expected SEMICOLON error 54: expected expression, item or let statement error 60: expected type @@ -160,6 +161,7 @@ error 65: expected pattern error 65: expected SEMICOLON error 65: expected expression, item or let statement error 92: expected expression +error 93: expected R_PAREN error 93: expected SEMICOLON error 93: expected expression, item or let statement error 95: expected expression, item or let statement diff --git a/crates/parser/test_data/parser/err/0024_many_type_parens.rast b/crates/parser/test_data/parser/err/0024_many_type_parens.rast index a2cf225af8..f0dbc9b102 100644 --- a/crates/parser/test_data/parser/err/0024_many_type_parens.rast +++ b/crates/parser/test_data/parser/err/0024_many_type_parens.rast @@ -168,12 +168,12 @@ SOURCE_FILE L_PAREN "(" ERROR QUESTION "?" - EXPR_STMT - PATH_EXPR - PATH - PATH_SEGMENT - NAME_REF - IDENT "Sized" + TYPE_ARG + PATH_TYPE + PATH + PATH_SEGMENT + NAME_REF + IDENT "Sized" ERROR R_PAREN ")" WHITESPACE " " @@ -291,15 +291,13 @@ SOURCE_FILE WHITESPACE "\n" R_CURLY "}" WHITESPACE "\n" -error 88: expected COMMA error 88: expected R_ANGLE error 121: expected SEMICOLON error 121: expected expression, item or let statement error 140: expected type error 141: expected R_PAREN error 141: expected COMMA -error 141: expected R_ANGLE -error 141: expected SEMICOLON +error 146: expected R_ANGLE error 146: expected SEMICOLON error 146: expected expression, item or let statement error 148: expected expression, item or let statement @@ -309,7 +307,6 @@ error 165: expected expression error 168: expected expression error 179: expected expression error 180: expected SEMICOLON -error 215: expected COMMA error 215: expected R_ANGLE error 235: expected SEMICOLON error 235: expected expression, item or let statement diff --git a/crates/parser/test_data/parser/err/0025_nope.rast b/crates/parser/test_data/parser/err/0025_nope.rast index 6b49724ec9..b6bc008837 100644 --- a/crates/parser/test_data/parser/err/0025_nope.rast +++ b/crates/parser/test_data/parser/err/0025_nope.rast @@ -156,8 +156,7 @@ SOURCE_FILE PATH_SEGMENT NAME_REF IDENT "i32" - WHITESPACE " " - ERROR + WHITESPACE " " ERROR L_CURLY "{" R_CURLY "}" @@ -199,10 +198,8 @@ error 95: expected type error 95: expected COMMA error 96: expected field error 98: expected field declaration +error 371: expected R_PAREN error 371: expected COMMA -error 372: expected a type -error 372: expected R_PAREN -error 372: expected COMMA error 372: expected enum variant error 374: expected enum variant error 494: expected pattern diff --git a/crates/parser/test_data/parser/err/0042_weird_blocks.rast b/crates/parser/test_data/parser/err/0042_weird_blocks.rast index 9cea337ce9..1cdc6e6e71 100644 --- a/crates/parser/test_data/parser/err/0042_weird_blocks.rast +++ b/crates/parser/test_data/parser/err/0042_weird_blocks.rast @@ -72,4 +72,4 @@ SOURCE_FILE error 24: expected existential, fn, trait or impl error 41: expected existential, fn, trait or impl error 56: expected a block -error 75: expected a loop +error 75: expected a loop or block diff --git a/crates/parser/test_data/parser/err/0048_double_fish.rast b/crates/parser/test_data/parser/err/0048_double_fish.rast index 3a05bfee1e..207a5c24df 100644 --- a/crates/parser/test_data/parser/err/0048_double_fish.rast +++ b/crates/parser/test_data/parser/err/0048_double_fish.rast @@ -12,7 +12,7 @@ SOURCE_FILE STMT_LIST L_CURLY "{" WHITESPACE "\n " - EXPR_STMT + BIN_EXPR PATH_EXPR PATH PATH_SEGMENT @@ -41,13 +41,14 @@ SOURCE_FILE COLON2 "::" ERROR L_ANGLE "<" - BIN_EXPR - PATH_EXPR - PATH - PATH_SEGMENT - NAME_REF - IDENT "nope" - SHR ">>" + TYPE_ARG + PATH_TYPE + PATH + PATH_SEGMENT + NAME_REF + IDENT "nope" + R_ANGLE ">" + R_ANGLE ">" ERROR SEMICOLON ";" WHITESPACE "\n" @@ -114,8 +115,6 @@ SOURCE_FILE WHITESPACE "\n" error 30: expected identifier error 31: expected COMMA -error 31: expected R_ANGLE -error 31: expected SEMICOLON error 37: expected expression error 75: expected identifier error 76: expected SEMICOLON diff --git a/crates/parser/test_data/parser/inline/err/0002_misplaced_label_err.rast b/crates/parser/test_data/parser/inline/err/0002_misplaced_label_err.rast index 56cea4b156..ea5203fb96 100644 --- a/crates/parser/test_data/parser/inline/err/0002_misplaced_label_err.rast +++ b/crates/parser/test_data/parser/inline/err/0002_misplaced_label_err.rast @@ -23,6 +23,6 @@ SOURCE_FILE WHITESPACE "\n" R_CURLY "}" WHITESPACE "\n" -error 22: expected a loop +error 22: expected a loop or block error 27: expected type error 27: expected `{` diff --git a/crates/parser/test_data/parser/inline/err/0015_arg_list_recovery.rast b/crates/parser/test_data/parser/inline/err/0015_arg_list_recovery.rast index 6e065f8f80..5d0fe859c2 100644 --- a/crates/parser/test_data/parser/inline/err/0015_arg_list_recovery.rast +++ b/crates/parser/test_data/parser/inline/err/0015_arg_list_recovery.rast @@ -72,6 +72,6 @@ SOURCE_FILE R_CURLY "}" WHITESPACE "\n" error 25: expected identifier -error 39: expected `,` +error 39: expected COMMA error 39: expected expression error 55: expected expression diff --git a/crates/parser/test_data/parser/inline/err/0015_missing_fn_param_type.rast b/crates/parser/test_data/parser/inline/err/0015_missing_fn_param_type.rast index e72df374d1..ea50ad35d7 100644 --- a/crates/parser/test_data/parser/inline/err/0015_missing_fn_param_type.rast +++ b/crates/parser/test_data/parser/inline/err/0015_missing_fn_param_type.rast @@ -49,5 +49,5 @@ SOURCE_FILE R_CURLY "}" WHITESPACE "\n" error 6: missing type for function parameter -error 6: expected COMMA +error 6: expected `,` error 16: missing type for function parameter |