+ /* error: expected expression */;

+ /* error: expected expression, expected COMMA */;

+ /* error: expected expression */::core::fmt::Arguments::new_v1(&["", ], &[::core::fmt::ArgumentV1::new(&(), ::core::fmt::Display::fmt), ]);

+ /* error: expected expression, expected R_PAREN */;

+ /* error: expected field name or number *//* parse error: expected field name or number */

+

+#[test]

+fn token_mapping_eager() {

+ check(

+ r#"

+#[rustc_builtin_macro]

+#[macro_export]

+macro_rules! format_args {}

+

+macro_rules! identity {

+ ($expr:expr) => { $expr };

+}

+

+fn main(foo: ()) {

+ format_args/*+tokenids*/!("{} {} {}", format_args!("{}", 0), foo, identity!(10), "bar")

+}

+

+"#,

+ expect![[r##"

+#[rustc_builtin_macro]

+#[macro_export]

+macro_rules! format_args {}

+

+macro_rules! identity {

+ ($expr:expr) => { $expr };

+}

+

+fn main(foo: ()) {

+ // format_args/*+tokenids*/!("{} {} {}"#1,#3 format_args!("{}", 0#10),#12 foo#13,#14 identity!(10#18),#21 "bar"#22)

+::core#4294967295::fmt#4294967295::Arguments#4294967295::new_v1#4294967295(&#4294967295[#4294967295""#4294967295,#4294967295 " "#4294967295,#4294967295 " "#4294967295,#4294967295 ]#4294967295,#4294967295 &#4294967295[::core#4294967295::fmt#4294967295::ArgumentV1#4294967295::new#4294967295(&#4294967295(::core#4294967295::fmt#4294967295::Arguments#4294967295::new_v1#4294967295(&#4294967295[#4294967295""#4294967295,#4294967295 ]#4294967295,#4294967295 &#4294967295[::core#4294967295::fmt#4294967295::ArgumentV1#4294967295::new#4294967295(&#4294967295(#42949672950#10)#4294967295,#4294967295 ::core#4294967295::fmt#4294967295::Display#4294967295::fmt#4294967295)#4294967295,#4294967295 ]#4294967295)#4294967295)#4294967295,#4294967295 ::core#4294967295::fmt#4294967295::Display#4294967295::fmt#4294967295)#4294967295,#4294967295 ::core#4294967295::fmt#4294967295::ArgumentV1#4294967295::new#4294967295(&#4294967295(#4294967295foo#13)#4294967295,#4294967295 ::core#4294967295::fmt#4294967295::Display#4294967295::fmt#4294967295)#4294967295,#4294967295 ::core#4294967295::fmt#4294967295::ArgumentV1#4294967295::new#4294967295(&#4294967295(#429496729510#18)#4294967295,#4294967295 ::core#4294967295::fmt#4294967295::Display#4294967295::fmt#4294967295)#4294967295,#4294967295 ]#4294967295)#4294967295

+}

+

+"##]],

+ );

+}

+

+ if let Some((tree, map, _)) = arg.value.as_deref() {

+ let Some(EagerCallInfo { arg,arg_id, .. }) = loc.eager.as_deref() else {

+use mbe::{syntax_node_to_token_tree, ValueResult};

+ ) -> ValueResult<

+ Option<Arc<(tt::Subtree, mbe::TokenMap, fixup::SyntaxFixupUndoInfo)>>,

+ Arc<Box<[SyntaxError]>>,

+ >;

+ fn macro_arg_node(

+ &self,

+ id: MacroCallId,

+ ) -> ValueResult<Option<GreenNode>, Arc<Box<[SyntaxError]>>>;

+ /// Fetches the expander for this macro.

+ fn macro_expander(&self, id: MacroDefId) -> TokenExpander;

+ /// Fetches (and compiles) the expander of this decl macro.

+) -> ValueResult<

+ Option<Arc<(tt::Subtree, mbe::TokenMap, fixup::SyntaxFixupUndoInfo)>>,

+ Arc<Box<[SyntaxError]>>,

+> {

+ return ValueResult::ok(Some(Arc::new((arg.0.clone(), arg.1.clone(), Default::default()))));

+ let ValueResult { value, err } = db.macro_arg_node(id);

+ let Some(arg) = value else {

+ return ValueResult { value: None, err };

+ };

+ let val = Some(Arc::new((tt, tmap, fixups.undo_info)));

+ match err {

+ Some(err) => ValueResult::new(val, err),

+ None => ValueResult::ok(val),

+ }

+fn macro_arg_node(

+ db: &dyn ExpandDatabase,

+ id: MacroCallId,

+) -> ValueResult<Option<GreenNode>, Arc<Box<[SyntaxError]>>> {

+ let err = || -> Arc<Box<[_]>> {

+ Arc::new(Box::new([SyntaxError::new_at_offset(

+ "invalid macro call".to_owned(),

+ syntax::TextSize::from(0),

+ )]))

+ };

+ let arg = if let MacroDefKind::BuiltInEager(..) = loc.def.kind {

+ let res = if let Some(EagerCallInfo { arg, .. }) = loc.eager.as_deref() {

+ Some(mbe::token_tree_to_syntax_node(&arg.0, mbe::TopEntryPoint::Expr).0)

+ } else {

+ loc.kind

+ .arg(db)

+ .and_then(|arg| ast::TokenTree::cast(arg.value))

+ .map(|tt| tt.reparse_as_expr().to_syntax())

+ };

+

+ match res {

+ Some(res) if res.errors().is_empty() => res.syntax_node(),

+ Some(res) => {

+ return ValueResult::new(

+ Some(res.syntax_node().green().into()),

+ // Box::<[_]>::from(res.errors()), not stable yet

+ Arc::new(res.errors().to_vec().into_boxed_slice()),

+ );

+ }

+ None => return ValueResult::only_err(err()),

+ }

+ } else {

+ match loc.kind.arg(db) {

+ Some(res) => res.value,

+ None => return ValueResult::only_err(err()),

+ }

+ };

+ return ValueResult::only_err(Arc::new(Box::new([SyntaxError::new(

+ "unbalanced token tree".to_owned(),

+ arg.text_range(),

+ )])));

+ ValueResult::ok(Some(arg.green().into()))

+fn macro_expander(db: &dyn ExpandDatabase, id: MacroDefId) -> TokenExpander {

+ let ExpandResult { value: tt, mut err } = match loc.def.kind {

+ let arg = db.macro_arg_node(id).value.unwrap();

+ let mut res = expander.expand(db, id, &adt, &tmap);

+ fixup::reverse_fixups(&mut res.value, &tmap, &fixups.undo_info);

+ res

+ let ValueResult { value, err } = db.macro_arg(id);

+ let Some(macro_arg) = value else {

+

+ // This might look a bit odd, but we do not expand the inputs to eager macros here.

+ // Eager macros inputs are expanded, well, eagerly when we collect the macro calls.

+ // That kind of expansion uses the ast id map of an eager macros input though which goes through

+ // the HirFileId machinery. As eager macro inputs are assigned a macro file id that query

+ // will end up going through here again, whereas we want to just want to inspect the raw input.

+ // As such we just return the input subtree here.

+ MacroDefKind::BuiltInEager(..) if loc.eager.is_none() => {

+ let mut arg = arg.clone();

+ fixup::reverse_fixups(&mut arg, arg_tm, undo_info);

+

+ return ExpandResult {

+ value: Arc::new(arg),

+ err: err.map(|err| {

+ let mut buf = String::new();

+ for err in &**err {

+ use std::fmt::Write;

+ _ = write!(buf, "{}, ", err);

+ }

+ buf.pop();

+ buf.pop();

+ ExpandError::other(buf)

+ }),

+ };

+ }

+ res

+ let Some(macro_arg) = db.macro_arg(id).value else {

+ // FIXME: We should make sure to enforce an invariant that invalid macro

+ // calls do not reach this call path!

+

+use rustc_hash::FxHashMap;

+use syntax::{ted, Parse, SyntaxNode, TextRange, TextSize, WalkEvent};

+ // the expansion which the ast id map is built upon has no whitespace, so the offsets are wrong as macro_call is from the token tree that has whitespace!

+ eager: None,

+ let ExpandResult { value: (arg_exp, arg_exp_map), err: parse_err } =

+ db.parse_macro_expansion(arg_id.as_macro_file());

+ // we need this map here as the expansion of the eager input fake file loses whitespace ...

+ let mut ws_mapping = FxHashMap::default();

+ if let Some((_, tm, _)) = db.macro_arg(arg_id).value.as_deref() {

+ ws_mapping.extend(tm.entries().filter_map(|(id, range)| {

+ Some((arg_exp_map.first_range_by_token(id, syntax::SyntaxKind::TOMBSTONE)?, range))

+ }));

+ }

+ InFile::new(arg_id.as_file(), arg_exp.syntax_node()),

+ let err = parse_err.or(err);

+ let Some((expanded_eager_input, mapping)) = expanded_eager_input else {

+

+ let og_tmap = mbe::syntax_node_to_token_map(

+ macro_call.value.token_tree().expect("macro_arg_text succeeded").syntax(),

+ );

+

+ let (mut subtree, expanded_eager_input_token_map) =

+ mbe::syntax_node_to_token_tree(&expanded_eager_input);

+

+ // The tokenmap and ids of subtree point into the expanded syntax node, but that is inaccessible from the outside

+ // so we need to remap them to the original input of the eager macro.

+ subtree.visit_ids(&|id| {

+ // Note: we discard all token ids of braces and the like here, but that's not too bad and only a temporary fix

+

+ if let Some(range) =

+ expanded_eager_input_token_map.first_range_by_token(id, syntax::SyntaxKind::TOMBSTONE)

+ {

+ // remap from expanded eager input to eager input expansion

+ if let Some(og_range) = mapping.get(&range) {

+ // remap from eager input expansion to original eager input

+ if let Some(&og_range) = ws_mapping.get(og_range) {

+ if let Some(og_token) = og_tmap.token_by_range(og_range) {

+ return og_token;

+ }

+ }

+ }

+ }

+ tt::TokenId::UNSPECIFIED

+ });

+ arg: Arc::new((subtree, og_tmap)),

+ arg_id,

+) -> ExpandResult<(InFile<Parse<SyntaxNode>>, Arc<mbe::TokenMap>)> {

+ let ast_id = macro_call.with_value(ast_id);

+ let id = def.as_lazy_macro(db, krate, MacroCallKind::FnLike { ast_id, expand_to });

+ db.parse_macro_expansion(macro_file)

+ .map(|parse| (InFile::new(macro_file.into(), parse.0), parse.1))

+) -> Result<ExpandResult<Option<(SyntaxNode, FxHashMap<TextRange, TextRange>)>>, UnresolvedMacro> {

+ let mut mapping = FxHashMap::default();

+ let mut offset = 0i32;

+ let apply_offset = |it: TextSize, offset: i32| {

+ TextSize::from(u32::try_from(offset + u32::from(it) as i32).unwrap_or_default())

+ };

+ let mut children = original.preorder_with_tokens();

+ while let Some(child) = children.next() {

+ let WalkEvent::Enter(child) = child else { continue };

+ let call = match child {

+ syntax::NodeOrToken::Node(node) => match ast::MacroCall::cast(node) {

+ Some(it) => {

+ children.skip_subtree();

+ it

+ }

+ None => continue,

+ },

+ syntax::NodeOrToken::Token(t) => {

+ mapping.insert(

+ TextRange::new(

+ apply_offset(t.text_range().start(), offset),

+ apply_offset(t.text_range().end(), offset),

+ ),

+ t.text_range(),

+ );

+ continue;

+ }

+ };

+ let def = match call.path().and_then(|path| ModPath::from_src(db, path, hygiene)) {

+ curr.with_value(call.clone()),

+ Some(call_id) => {

+ db.parse_macro_expansion(call_id.as_macro_file());

+

+ let call_tt_start =

+ call.token_tree().unwrap().syntax().text_range().start();

+ let call_start = apply_offset(call.syntax().text_range().start(), offset);

+ if let Some((_, arg_map, _)) = db.macro_arg(call_id).value.as_deref() {

+ mapping.extend(arg_map.entries().filter_map(|(tid, range)| {

+ value

+ .1

+ .first_range_by_token(tid, syntax::SyntaxKind::TOMBSTONE)

+ .map(|r| (r + call_start, range + call_tt_start))

+ }));

+ };

+

+ let ExpandResult { value: (parse, tm), err } =

+ lazy_expand(db, &def, curr.with_value(call.clone()), krate);

+ let decl_mac = if let MacroDefKind::Declarative(ast_id) = def.kind {

+ Some(db.decl_macro_expander(def.krate, ast_id))

+ } else {

+ None

+ };

+ let hygiene = Hygiene::new(db, parse.file_id);

+ parse.as_ref().map(|it| it.syntax_node()),

+

+ let call_tt_start = call.token_tree().unwrap().syntax().text_range().start();

+ let call_start = apply_offset(call.syntax().text_range().start(), offset);

+ if let Some((_tt, arg_map, _)) = parse

+ .file_id

+ .macro_file()

+ .and_then(|id| db.macro_arg(id.macro_call_id).value)

+ .as_deref()

+ {

+ mapping.extend(arg_map.entries().filter_map(|(tid, range)| {

+ tm.first_range_by_token(

+ decl_mac.as_ref().map(|it| it.map_id_down(tid)).unwrap_or(tid),

+ syntax::SyntaxKind::TOMBSTONE,

+ )

+ .map(|r| (r + call_start, range + call_tt_start))

+ }));

+ };

+ // FIXME: Do we need to re-use _m here?

+ ExpandResult { value: value.map(|(n, _m)| n), err }

+ if call.syntax() == &original {

+ return Ok(ExpandResult { value: value.zip(Some(mapping)), err: error });

+ offset += u32::from(insert.text_range().len()) as i32

+ - u32::from(call.syntax().text_range().len()) as i32;

+ replacements.push((call, insert));

+ Ok(ExpandResult { value: Some((original, mapping)), err: error })

+ original: Box<[Subtree]>,

+ undo_info: SyntaxFixupUndoInfo { original: original.into_boxed_slice() },

+ let macro_def = db.macro_expander(loc.def);

+ let macro_arg = db.macro_arg(macro_file.macro_call_id).value.unwrap_or_else(|| {

+ /// Some if this is a macro call for an eager macro. Note that this is `None`

+ /// for the eager input macro file.

+ /// The expanded argument of the eager macro.

+ arg_id: MacroCallId,

+ let is_include_expansion = loc.def.is_include() && loc.eager.is_some();

+ matches!(loc.def.kind, MacroDefKind::BuiltInEager(..))

+ if loc.def.is_include() {

+ let macro_def = db.macro_expander(loc.def);

+ let macro_arg = db.macro_arg(macro_file.macro_call_id).value.unwrap_or_else(|| {

+ Arc::new((

+ tt::Subtree { delimiter: tt::Delimiter::UNSPECIFIED, token_trees: Vec::new() },

+ Default::default(),

+ Default::default(),

+ ))

+ });

+ ExpandProcMacroQuery, HygieneFrameQuery, InternMacroCallQuery, MacroArgNodeQuery,

+ if let hir_expand::db::TokenExpander::DeclarativeMacro(expander) = db.macro_expander(id) {

+ hir::db::MacroArgNodeQuery

+ hir_db::MacroArgNodeQuery

+ <span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis macro">(</span><span class="string_literal macro">"Hello </span><span class="format_specifier">{</span><span class="format_specifier">:</span><span class="numeric_literal">0</span><span class="numeric_literal">5</span><span class="format_specifier">}</span><span class="string_literal macro">!"</span><span class="comma macro">,</span> <span class="numeric_literal macro">-</span><span class="numeric_literal macro">5</span><span class="parenthesis macro">)</span><span class="semicolon">;</span>

+ <span class="keyword">let</span> <span class="variable declaration reference">backslash</span> <span class="operator">=</span> <span class="string_literal">r"\\"</span><span class="semicolon">;</span>

+ <span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis macro">(</span><span class="string_literal macro">"</span><span class="format_specifier">{</span><span class="format_specifier">:</span><span class="variable">x</span><span class="format_specifier">?</span><span class="format_specifier">}</span><span class="string_literal macro"> </span><span class="format_specifier">{</span><span class="format_specifier">}</span><span class="string_literal macro"> "</span><span class="comma macro">,</span> <span class="unresolved_reference macro">thingy</span><span class="comma macro">,</span> <span class="unresolved_reference macro">n2</span><span class="parenthesis macro">)</span><span class="semicolon">;</span>

+ <span class="macro">format_args</span><span class="macro_bang">!</span><span class="parenthesis macro">(</span><span class="none macro">concat</span><span class="punctuation macro">!</span><span class="parenthesis macro">(</span><span class="string_literal macro">"</span><span class="format_specifier">{</span><span class="format_specifier">}</span><span class="string_literal macro">"</span><span class="parenthesis macro">)</span><span class="comma macro">,</span> <span class="string_literal macro">"{}"</span><span class="parenthesis macro">)</span><span class="semicolon">;</span>

+ <span class="macro">format_args</span><span class="macro_bang">!</span><span class="parenthesis macro">(</span><span class="string_literal macro">"</span><span class="format_specifier">{</span><span class="format_specifier">}</span><span class="string_literal macro"> </span><span class="format_specifier">{</span><span class="format_specifier">}</span><span class="string_literal macro"> </span><span class="format_specifier">{</span><span class="format_specifier">}</span><span class="string_literal macro"> </span><span class="format_specifier">{</span><span class="format_specifier">}</span><span class="string_literal macro"> </span><span class="format_specifier">{</span><span class="format_specifier">}</span><span class="string_literal macro"> </span><span class="format_specifier">{</span><span class="format_specifier">}</span><span class="string_literal macro">"</span><span class="comma macro">,</span> <span class="none macro">backslash</span><span class="comma macro">,</span> <span class="none macro">format_args</span><span class="punctuation macro">!</span><span class="parenthesis macro">(</span><span class="string_literal macro">"</span><span class="format_specifier">{</span><span class="format_specifier">}</span><span class="string_literal macro">"</span><span class="comma macro">,</span> <span class="numeric_literal macro">0</span><span class="parenthesis macro">)</span><span class="comma macro">,</span> <span class="none macro">foo</span><span class="comma macro">,</span> <span class="string_literal macro">"bar"</span><span class="comma macro">,</span> <span class="none macro">toho</span><span class="punctuation macro">!</span><span class="parenthesis macro">(</span><span class="parenthesis macro">)</span><span class="comma macro">,</span> <span class="none macro">backslash</span><span class="parenthesis macro">)</span><span class="semicolon">;</span>

+ let backslash = r"\\";

+ format_args!("{} {} {} {} {} {}", backslash, format_args!("{}", 0), foo, "bar", toho!(), backslash);

+

+ pub fn entries(&self) -> impl Iterator<Item = (tt::TokenId, TextRange)> + '_ {

+ self.entries.iter().filter_map(|&(tid, tr)| match tr {

+ TokenTextRange::Token(range) => Some((tid, range)),

+ TokenTextRange::Delimiter(_) => None,

+ })

+ }

+impl ast::TokenTree {

+ pub fn reparse_as_expr(self) -> Parse<ast::Expr> {

+ let tokens = self.syntax().descendants_with_tokens().filter_map(NodeOrToken::into_token);

+

+ let mut parser_input = parser::Input::default();

+ let mut was_joint = false;

+ for t in tokens {

+ let kind = t.kind();

+ if kind.is_trivia() {

+ was_joint = false

+ } else {

+ if kind == SyntaxKind::IDENT {

+ let token_text = t.text();

+ let contextual_kw = SyntaxKind::from_contextual_keyword(token_text)

+ .unwrap_or(SyntaxKind::IDENT);

+ parser_input.push_ident(contextual_kw);

+ } else {

+ if was_joint {

+ parser_input.was_joint();

+ }

+ parser_input.push(kind);

+ // Tag the token as joint if it is float with a fractional part

+ // we use this jointness to inform the parser about what token split

+ // event to emit when we encounter a float literal in a field access

+ if kind == SyntaxKind::FLOAT_NUMBER && !t.text().ends_with('.') {

+ parser_input.was_joint();

+ }

+ }

+ was_joint = true;

+ }

+ }

+

+ let parser_output = parser::TopEntryPoint::Expr.parse(&parser_input);

+

+ let mut tokens =

+ self.syntax().descendants_with_tokens().filter_map(NodeOrToken::into_token);

+ let mut text = String::new();

+ let mut pos = TextSize::from(0);

+ let mut builder = SyntaxTreeBuilder::default();

+ for event in parser_output.iter() {

+ match event {

+ parser::Step::Token { kind, n_input_tokens } => {

+ let mut token = tokens.next().unwrap();

+ while token.kind().is_trivia() {

+ let text = token.text();

+ pos += TextSize::from(text.len() as u32);

+ builder.token(token.kind(), text);

+

+ token = tokens.next().unwrap();

+ }

+ text.push_str(token.text());

+ for _ in 1..n_input_tokens {

+ let token = tokens.next().unwrap();

+ text.push_str(token.text());

+ }

+

+ pos += TextSize::from(text.len() as u32);

+ builder.token(kind, &text);

+ text.clear();

+ }

+ parser::Step::FloatSplit { ends_in_dot: has_pseudo_dot } => {

+ let token = tokens.next().unwrap();

+ let text = token.text();

+

+ match text.split_once('.') {

+ Some((left, right)) => {

+ assert!(!left.is_empty());

+ builder.start_node(SyntaxKind::NAME_REF);

+ builder.token(SyntaxKind::INT_NUMBER, left);

+ builder.finish_node();

+

+ // here we move the exit up, the original exit has been deleted in process

+ builder.finish_node();

+

+ builder.token(SyntaxKind::DOT, ".");

+

+ if has_pseudo_dot {

+ assert!(right.is_empty(), "{left}.{right}");

+ } else {

+ builder.start_node(SyntaxKind::NAME_REF);

+ builder.token(SyntaxKind::INT_NUMBER, right);

+ builder.finish_node();

+

+ // the parser creates an unbalanced start node, we are required to close it here

+ builder.finish_node();

+ }

+ }

+ None => unreachable!(),

+ }

+ pos += TextSize::from(text.len() as u32);

+ }

+ parser::Step::Enter { kind } => builder.start_node(kind),

+ parser::Step::Exit => builder.finish_node(),

+ parser::Step::Error { msg } => builder.error(msg.to_owned(), pos),

+ }

+ }

+

+ let (green, errors) = builder.finish_raw();

+

+ Parse { green, errors: Arc::new(errors), _ty: PhantomData }

+ }

+}

+

+

+ impl Subtree {

+ pub fn visit_ids(&mut self, f: &impl Fn(TokenId) -> TokenId) {

+ self.delimiter.open = f(self.delimiter.open);

+ self.delimiter.close = f(self.delimiter.close);

+ self.token_trees.iter_mut().for_each(|tt| match tt {

+ crate::TokenTree::Leaf(leaf) => match leaf {

+ crate::Leaf::Literal(it) => it.span = f(it.span),

+ crate::Leaf::Punct(it) => it.span = f(it.span),

+ crate::Leaf::Ident(it) => it.span = f(it.span),

+ },

+ crate::TokenTree::Subtree(s) => s.visit_ids(f),

+ })

+ }

+ }