Unnamed repository; edit this file 'description' to name the repository.
Auto merge of #18151 - ChayimFriedman2:metavar-concat, r=Veykril
feat: Support the `${concat(...)}` metavariable expression
I didn't follow rustc precisely, because I think it does some things wrongly (or they are FIXME), but I only allowed more code, not less. So we're all fine.
Closes #18145.
| -rw-r--r-- | Cargo.lock | 1 | ||||
| -rw-r--r-- | crates/hir-def/src/macro_expansion_tests/mbe/metavar_expr.rs | 147 | ||||
| -rw-r--r-- | crates/mbe/Cargo.toml | 1 | ||||
| -rw-r--r-- | crates/mbe/src/benchmark.rs | 6 | ||||
| -rw-r--r-- | crates/mbe/src/expander/matcher.rs | 12 | ||||
| -rw-r--r-- | crates/mbe/src/expander/transcriber.rs | 80 | ||||
| -rw-r--r-- | crates/mbe/src/lib.rs | 5 | ||||
| -rw-r--r-- | crates/mbe/src/parser.rs | 50 | ||||
| -rw-r--r-- | crates/tt/src/iter.rs | 7 |
9 files changed, 304 insertions, 5 deletions
diff --git a/Cargo.lock b/Cargo.lock index 6df73c2fd1..5df48778e3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1047,6 +1047,7 @@ dependencies = [ "expect-test", "intern", "parser", + "ra-ap-rustc_lexer", "rustc-hash", "smallvec", "span", diff --git a/crates/hir-def/src/macro_expansion_tests/mbe/metavar_expr.rs b/crates/hir-def/src/macro_expansion_tests/mbe/metavar_expr.rs index bf70119838..1430e2a9a9 100644 --- a/crates/hir-def/src/macro_expansion_tests/mbe/metavar_expr.rs +++ b/crates/hir-def/src/macro_expansion_tests/mbe/metavar_expr.rs @@ -311,3 +311,150 @@ fn test() { "#]], ); } + +#[test] +fn concat() { + // FIXME: Should this error? rustc currently accepts it. + check( + r#" +macro_rules! m { + ( $a:ident, $b:literal ) => { + let ${concat($a, _, "123", _foo, $b, _, 123)}; + }; +} + +fn test() { + m!( abc, 456 ); + m!( def, "hello" ); +} +"#, + expect![[r#" +macro_rules! m { + ( $a:ident, $b:literal ) => { + let ${concat($a, _, "123", _foo, $b, _, 123)}; + }; +} + +fn test() { + let abc_123_foo456_123;; + let def_123_foohello_123;; +} +"#]], + ); +} + +#[test] +fn concat_less_than_two_elements() { + // FIXME: Should this error? rustc currently accepts it. + check( + r#" +macro_rules! m { + () => { + let ${concat(abc)}; + }; +} + +fn test() { + m!() +} +"#, + expect![[r#" +macro_rules! m { + () => { + let ${concat(abc)}; + }; +} + +fn test() { + /* error: macro definition has parse errors */ +} +"#]], + ); +} + +#[test] +fn concat_invalid_ident() { + // FIXME: Should this error? rustc currently accepts it. + check( + r#" +macro_rules! m { + () => { + let ${concat(abc, '"')}; + }; +} + +fn test() { + m!() +} +"#, + expect![[r#" +macro_rules! m { + () => { + let ${concat(abc, '"')}; + }; +} + +fn test() { + /* error: `${concat(..)}` is not generating a valid identifier */let __ra_concat_dummy; +} +"#]], + ); +} + +#[test] +fn concat_invalid_fragment() { + // FIXME: Should this error? rustc currently accepts it. + check( + r#" +macro_rules! m { + ( $e:expr ) => { + let ${concat(abc, $e)}; + }; +} + +fn test() { + m!(()) +} +"#, + expect![[r#" +macro_rules! m { + ( $e:expr ) => { + let ${concat(abc, $e)}; + }; +} + +fn test() { + /* error: metavariables of `${concat(..)}` must be of type `ident`, `literal` or `tt` */let abc; +} +"#]], + ); +} + +#[test] +fn concat_repetition() { + // FIXME: Should this error? rustc currently accepts it. + check( + r#" +macro_rules! m { + ( $($i:ident)* ) => { + let ${concat(abc, $i)}; + }; +} + +fn test() { + m!(a b c) +} +"#, + expect![[r#" +macro_rules! m { + ( $($i:ident)* ) => { + let ${concat(abc, $i)}; + }; +} + +fn test() { + /* error: expected simple binding, found nested binding `i` */let abc; +} +"#]], + ); +} diff --git a/crates/mbe/Cargo.toml b/crates/mbe/Cargo.toml index 413a0254a6..b37d57f280 100644 --- a/crates/mbe/Cargo.toml +++ b/crates/mbe/Cargo.toml @@ -18,6 +18,7 @@ rustc-hash.workspace = true smallvec.workspace = true tracing.workspace = true arrayvec.workspace = true +ra-ap-rustc_lexer.workspace = true # local deps syntax.workspace = true diff --git a/crates/mbe/src/benchmark.rs b/crates/mbe/src/benchmark.rs index f8c1d027c6..9e4ef14074 100644 --- a/crates/mbe/src/benchmark.rs +++ b/crates/mbe/src/benchmark.rs @@ -216,7 +216,11 @@ fn invocation_fixtures( token_trees.push(subtree.into()); } - Op::Ignore { .. } | Op::Index { .. } | Op::Count { .. } | Op::Len { .. } => {} + Op::Ignore { .. } + | Op::Index { .. } + | Op::Count { .. } + | Op::Len { .. } + | Op::Concat { .. } => {} }; // Simple linear congruential generator for deterministic result diff --git a/crates/mbe/src/expander/matcher.rs b/crates/mbe/src/expander/matcher.rs index 4a052bbb9f..95641abc0f 100644 --- a/crates/mbe/src/expander/matcher.rs +++ b/crates/mbe/src/expander/matcher.rs @@ -584,7 +584,11 @@ fn match_loop_inner<'t>( error_items.push(item); } OpDelimited::Op( - Op::Ignore { .. } | Op::Index { .. } | Op::Count { .. } | Op::Len { .. }, + Op::Ignore { .. } + | Op::Index { .. } + | Op::Count { .. } + | Op::Len { .. } + | Op::Concat { .. }, ) => { stdx::never!("metavariable expression in lhs found"); } @@ -879,7 +883,11 @@ fn collect_vars(collector_fun: &mut impl FnMut(Symbol), pattern: &MetaTemplate) Op::Subtree { tokens, .. } => collect_vars(collector_fun, tokens), Op::Repeat { tokens, .. } => collect_vars(collector_fun, tokens), Op::Literal(_) | Op::Ident(_) | Op::Punct(_) => {} - Op::Ignore { .. } | Op::Index { .. } | Op::Count { .. } | Op::Len { .. } => { + Op::Ignore { .. } + | Op::Index { .. } + | Op::Count { .. } + | Op::Len { .. } + | Op::Concat { .. } => { stdx::never!("metavariable expression in lhs found"); } } diff --git a/crates/mbe/src/expander/transcriber.rs b/crates/mbe/src/expander/transcriber.rs index 0084411535..1db2f35d26 100644 --- a/crates/mbe/src/expander/transcriber.rs +++ b/crates/mbe/src/expander/transcriber.rs @@ -2,12 +2,12 @@ //! `$ident => foo`, interpolates variables in the template, to get `fn foo() {}` use intern::{sym, Symbol}; -use span::Span; +use span::{Edition, Span}; use tt::Delimiter; use crate::{ expander::{Binding, Bindings, Fragment}, - parser::{MetaVarKind, Op, RepeatKind, Separator}, + parser::{ConcatMetaVarExprElem, MetaVarKind, Op, RepeatKind, Separator}, ExpandError, ExpandErrorKind, ExpandResult, MetaTemplate, }; @@ -312,6 +312,82 @@ fn expand_subtree( .into(), ); } + Op::Concat { elements, span: concat_span } => { + let mut concatenated = String::new(); + for element in elements { + match element { + ConcatMetaVarExprElem::Ident(ident) => { + concatenated.push_str(ident.sym.as_str()) + } + ConcatMetaVarExprElem::Literal(lit) => { + // FIXME: This isn't really correct wrt. escaping, but that's what rustc does and anyway + // escaping is used most of the times for characters that are invalid in identifiers. + concatenated.push_str(lit.symbol.as_str()) + } + ConcatMetaVarExprElem::Var(var) => { + // Handling of repetitions in `${concat}` isn't fleshed out in rustc, so we currently + // err at it. + // FIXME: Do what rustc does for repetitions. + let var_value = match ctx.bindings.get_fragment( + &var.sym, + var.span, + &mut ctx.nesting, + marker, + ) { + Ok(var) => var, + Err(e) => { + if err.is_none() { + err = Some(e); + }; + continue; + } + }; + let value = match &var_value { + Fragment::Tokens(tt::TokenTree::Leaf(tt::Leaf::Ident(ident))) => { + ident.sym.as_str() + } + Fragment::Tokens(tt::TokenTree::Leaf(tt::Leaf::Literal(lit))) => { + lit.symbol.as_str() + } + _ => { + if err.is_none() { + err = Some(ExpandError::binding_error(var.span, "metavariables of `${concat(..)}` must be of type `ident`, `literal` or `tt`")) + } + continue; + } + }; + concatenated.push_str(value); + } + } + } + + // `${concat}` span comes from the macro (at least for now). + // See https://github.com/rust-lang/rust/blob/b0af276da341/compiler/rustc_expand/src/mbe/transcribe.rs#L724-L726. + let mut result_span = *concat_span; + marker(&mut result_span); + + // FIXME: NFC normalize the result. + if !rustc_lexer::is_ident(&concatenated) { + if err.is_none() { + err = Some(ExpandError::binding_error( + *concat_span, + "`${concat(..)}` is not generating a valid identifier", + )); + } + // Insert a dummy identifier for better parsing. + concatenated.clear(); + concatenated.push_str("__ra_concat_dummy"); + } + + let needs_raw = + parser::SyntaxKind::from_keyword(&concatenated, Edition::LATEST).is_some(); + let is_raw = if needs_raw { tt::IdentIsRaw::Yes } else { tt::IdentIsRaw::No }; + arena.push(tt::TokenTree::Leaf(tt::Leaf::Ident(tt::Ident { + is_raw, + span: result_span, + sym: Symbol::intern(&concatenated), + }))); + } } } // drain the elements added in this instance of expand_subtree diff --git a/crates/mbe/src/lib.rs b/crates/mbe/src/lib.rs index f7b6e8f54b..dd71e46db3 100644 --- a/crates/mbe/src/lib.rs +++ b/crates/mbe/src/lib.rs @@ -6,6 +6,11 @@ //! The tests for this functionality live in another crate: //! `hir_def::macro_expansion_tests::mbe`. +#[cfg(not(feature = "in-rust-tree"))] +extern crate ra_ap_rustc_lexer as rustc_lexer; +#[cfg(feature = "in-rust-tree")] +extern crate rustc_lexer; + mod expander; mod parser; diff --git a/crates/mbe/src/parser.rs b/crates/mbe/src/parser.rs index 2efe318f61..b55edf4a5e 100644 --- a/crates/mbe/src/parser.rs +++ b/crates/mbe/src/parser.rs @@ -84,6 +84,10 @@ pub(crate) enum Op { // FIXME: `usize`` once we drop support for 1.76 depth: Option<usize>, }, + Concat { + elements: Box<[ConcatMetaVarExprElem]>, + span: Span, + }, Repeat { tokens: MetaTemplate, kind: RepeatKind, @@ -98,6 +102,18 @@ pub(crate) enum Op { Ident(tt::Ident<Span>), } +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) enum ConcatMetaVarExprElem { + /// There is NO preceding dollar sign, which means that this identifier should be interpreted + /// as a literal. + Ident(tt::Ident<Span>), + /// There is a preceding dollar sign, which means that this identifier should be expanded + /// and interpreted as a variable. + Var(tt::Ident<Span>), + /// For example, a number or a string. + Literal(tt::Literal<Span>), +} + #[derive(Copy, Clone, Debug, PartialEq, Eq)] pub(crate) enum RepeatKind { ZeroOrMore, @@ -384,6 +400,32 @@ fn parse_metavar_expr(src: &mut TtIter<'_, Span>) -> Result<Op, ()> { let depth = if try_eat_comma(&mut args) { Some(parse_depth(&mut args)?) } else { None }; Op::Count { name: ident.sym.clone(), depth } } + s if sym::concat == *s => { + let mut elements = Vec::new(); + while let Some(next) = args.peek_n(0) { + let element = if let tt::TokenTree::Leaf(tt::Leaf::Literal(lit)) = next { + args.next().expect("already peeked"); + ConcatMetaVarExprElem::Literal(lit.clone()) + } else { + let is_var = try_eat_dollar(&mut args); + let ident = args.expect_ident_or_underscore()?.clone(); + + if is_var { + ConcatMetaVarExprElem::Var(ident) + } else { + ConcatMetaVarExprElem::Ident(ident) + } + }; + elements.push(element); + if args.peek_n(0).is_some() { + args.expect_comma()?; + } + } + if elements.len() < 2 { + return Err(()); + } + Op::Concat { elements: elements.into_boxed_slice(), span: func.span } + } _ => return Err(()), }; @@ -414,3 +456,11 @@ fn try_eat_comma(src: &mut TtIter<'_, Span>) -> bool { } false } + +fn try_eat_dollar(src: &mut TtIter<'_, Span>) -> bool { + if let Some(tt::TokenTree::Leaf(tt::Leaf::Punct(tt::Punct { char: '$', .. }))) = src.peek_n(0) { + let _ = src.next(); + return true; + } + false +} diff --git a/crates/tt/src/iter.rs b/crates/tt/src/iter.rs index e96bed0319..587b903aa9 100644 --- a/crates/tt/src/iter.rs +++ b/crates/tt/src/iter.rs @@ -57,6 +57,13 @@ impl<'a, S: Copy> TtIter<'a, S> { } } + pub fn expect_comma(&mut self) -> Result<(), ()> { + match self.expect_leaf()? { + Leaf::Punct(Punct { char: ',', .. }) => Ok(()), + _ => Err(()), + } + } + pub fn expect_ident(&mut self) -> Result<&'a Ident<S>, ()> { match self.expect_leaf()? { Leaf::Ident(it) if it.sym != sym::underscore => Ok(it), |