rust-analyzer

diff --git a/crates/cfg/src/cfg_expr.rs b/crates/cfg/src/cfg_expr.rs
index c5da8443a6..d253f6f492 100644
--- a/crates/cfg/src/cfg_expr.rs
+++ b/crates/cfg/src/cfg_expr.rs

@@ -157,7 +157,8 @@ fn next_cfg_expr_from_ast(

ctx: span::SyntaxContext::root(span::Edition::Edition2015),

};

- let literal = tt::token_to_literal(literal.text(), dummy_span).symbol;

+ let literal =

+ Symbol::intern(tt::token_to_literal(literal.text(), dummy_span).text());

it.next();

CfgAtom::KeyValue { key: name, value: literal.clone() }.into()

} else {

@@ -211,7 +212,7 @@ fn next_cfg_expr(it: &mut tt::iter::TtIter<'_>) -> Option<CfgExpr> {

Some(tt::TtElement::Leaf(tt::Leaf::Literal(literal))) => {

it.next();

- CfgAtom::KeyValue { key: name, value: literal.symbol.clone() }.into()

+ CfgAtom::KeyValue { key: name, value: Symbol::intern(literal.text()) }.into()

}

_ => return Some(CfgExpr::Invalid),

}

diff --git a/crates/hir-def/src/item_tree/attrs.rs b/crates/hir-def/src/item_tree/attrs.rs
index bfebce0013..7907611284 100644
--- a/crates/hir-def/src/item_tree/attrs.rs
+++ b/crates/hir-def/src/item_tree/attrs.rs

@@ -226,7 +226,7 @@ impl<'attr> AttrQuery<'attr> {

}

#[inline]

- pub(crate) fn string_value_with_span(self) -> Option<(&'attr Symbol, span::Span)> {

+ pub(crate) fn string_value_with_span(self) -> Option<(&'attr str, span::Span)> {

self.attrs().find_map(|attr| attr.string_value_with_span())

}

diff --git a/crates/hir-def/src/nameres/collector.rs b/crates/hir-def/src/nameres/collector.rs
index 822da678d7..4740e3b99e 100644
--- a/crates/hir-def/src/nameres/collector.rs
+++ b/crates/hir-def/src/nameres/collector.rs

@@ -17,7 +17,7 @@ use hir_expand::{

name::{AsName, Name},

proc_macro::CustomProcMacroExpander,

};

-use intern::{Interned, sym};

+use intern::{Interned, Symbol, sym};

use itertools::izip;

use la_arena::Idx;

use rustc_hash::{FxHashMap, FxHashSet};

@@ -292,13 +292,13 @@ impl<'db> DefCollector<'db> {

match () {

() if *attr_name == sym::recursion_limit => {

if let Some(limit) = attr.string_value()

- && let Ok(limit) = limit.as_str().parse()

+ && let Ok(limit) = limit.parse()

{

crate_data.recursion_limit = Some(limit);

}

() if *attr_name == sym::crate_type => {

- if attr.string_value() == Some(&sym::proc_dash_macro) {

+ if attr.string_value() == Some("proc-macro") {

self.is_proc_macro = true;

}

@@ -2460,7 +2460,7 @@ impl ModCollector<'_, '_> {

let name;

let name = match attrs.by_key(sym::rustc_builtin_macro).string_value_with_span() {

Some((it, span)) => {

- name = Name::new_symbol(it.clone(), span.ctx);

+ name = Name::new_symbol(Symbol::intern(it), span.ctx);

&name

}

None => {

diff --git a/crates/hir-expand/src/attrs.rs b/crates/hir-expand/src/attrs.rs
index 5a44018720..e3f10b2129 100644
--- a/crates/hir-expand/src/attrs.rs
+++ b/crates/hir-expand/src/attrs.rs

@@ -35,7 +35,7 @@ use arrayvec::ArrayVec;

use base_db::Crate;

use cfg::{CfgExpr, CfgOptions};

use either::Either;

-use intern::{Interned, Symbol};

+use intern::Interned;

use itertools::Itertools;

use mbe::{DelimiterKind, Punct};

use parser::T;

@@ -417,37 +417,32 @@ impl fmt::Display for AttrInput {

impl Attr {

/// #[path = "string"]

- pub fn string_value(&self) -> Option<&Symbol> {

+ pub fn string_value(&self) -> Option<&str> {

match self.input.as_deref()? {

- AttrInput::Literal(tt::Literal {

- symbol: text,

- kind: tt::LitKind::Str | tt::LitKind::StrRaw(_),

- ..

- }) => Some(text),

+ AttrInput::Literal(

+ lit @ tt::Literal { kind: tt::LitKind::Str | tt::LitKind::StrRaw(_), .. },

+ ) => Some(lit.text()),

_ => None,

}

/// #[path = "string"]

- pub fn string_value_with_span(&self) -> Option<(&Symbol, span::Span)> {

+ pub fn string_value_with_span(&self) -> Option<(&str, span::Span)> {

match self.input.as_deref()? {

- AttrInput::Literal(tt::Literal {

- symbol: text,

- kind: tt::LitKind::Str | tt::LitKind::StrRaw(_),

- span,

- suffix: _,

- }) => Some((text, *span)),

+ AttrInput::Literal(

+ lit @ tt::Literal { kind: tt::LitKind::Str | tt::LitKind::StrRaw(_), span, .. },

+ ) => Some((lit.text(), *span)),

_ => None,

}

pub fn string_value_unescape(&self) -> Option<Cow<'_, str>> {

match self.input.as_deref()? {

- AttrInput::Literal(tt::Literal {

- symbol: text, kind: tt::LitKind::StrRaw(_), ..

- }) => Some(Cow::Borrowed(text.as_str())),

- AttrInput::Literal(tt::Literal { symbol: text, kind: tt::LitKind::Str, .. }) => {

- unescape(text.as_str())

+ AttrInput::Literal(lit @ tt::Literal { kind: tt::LitKind::StrRaw(_), .. }) => {

+ Some(Cow::Borrowed(lit.text()))

+ }

+ AttrInput::Literal(lit @ tt::Literal { kind: tt::LitKind::Str, .. }) => {

+ unescape(lit.text())

}

_ => None,

}

diff --git a/crates/hir-expand/src/builtin/fn_macro.rs b/crates/hir-expand/src/builtin/fn_macro.rs
index 011a9530c4..6e4b96b050 100644
--- a/crates/hir-expand/src/builtin/fn_macro.rs
+++ b/crates/hir-expand/src/builtin/fn_macro.rs

@@ -1,5 +1,7 @@

//! Builtin macro

+use std::borrow::Cow;

use base_db::AnchoredPath;

use cfg::CfgExpr;

use either::Either;

@@ -13,7 +15,7 @@ use span::{Edition, FileId, Span};

use stdx::format_to;

use syntax::{

format_smolstr,

- unescape::{unescape_byte, unescape_char, unescape_str},

+ unescape::{unescape_byte, unescape_char},

};

use syntax_bridge::syntax_node_to_token_tree;

@@ -177,12 +179,7 @@ fn line_expand(

// not incremental

ExpandResult::ok(tt::TopSubtree::invisible_from_leaves(

span,

- [tt::Leaf::Literal(tt::Literal {

- symbol: sym::INTEGER_0,

- span,

- kind: tt::LitKind::Integer,

- suffix: Some(sym::u32),

- })],

+ [tt::Leaf::Literal(tt::Literal::new("0", span, tt::LitKind::Integer, "u32"))],

))

}

@@ -303,7 +300,8 @@ fn format_args_nl_expand(

mut lit @ tt::Literal { kind: tt::LitKind::Str, .. },

))) = lit

{

- lit.symbol = Symbol::intern(&format_smolstr!("{}\\n", lit.symbol.as_str()));

+ let (text, suffix) = lit.text_and_suffix();

+ lit.text_and_suffix = Symbol::intern(&format_smolstr!("{text}\\n{suffix}"));

tt.set_token(1, lit.into());

}

ExpandResult::ok(quote! {span =>

@@ -521,14 +519,11 @@ fn compile_error_expand(

let err = match tt.iter().collect_array() {

Some(

[

- tt::TtElement::Leaf(tt::Leaf::Literal(tt::Literal {

- symbol: text,

- span: _,

- kind: tt::LitKind::Str | tt::LitKind::StrRaw(_),

- suffix: _,

- })),

+ tt::TtElement::Leaf(tt::Leaf::Literal(

+ lit @ tt::Literal { kind: tt::LitKind::Str | tt::LitKind::StrRaw(_), .. },

+ )),

- ) => ExpandError::other(span, Box::from(unescape_symbol(&text).as_str())),

+ ) => ExpandError::other(span, Box::from(unescape_str(lit.text()))),

_ => ExpandError::other(span, "`compile_error!` argument must be a string"),

};

@@ -569,20 +564,20 @@ fn concat_expand(

// as-is.

match it.kind {

tt::LitKind::Char => {

- if let Ok(c) = unescape_char(it.symbol.as_str()) {

+ if let Ok(c) = unescape_char(it.text()) {

text.push(c);

}

record_span(it.span);

}

tt::LitKind::Integer | tt::LitKind::Float => {

- format_to!(text, "{}", it.symbol.as_str())

+ format_to!(text, "{}", it.text())

}

tt::LitKind::Str => {

- text.push_str(unescape_symbol(&it.symbol).as_str());

+ text.push_str(&unescape_str(it.text()));

record_span(it.span);

}

tt::LitKind::StrRaw(_) => {

- format_to!(text, "{}", it.symbol.as_str());

+ format_to!(text, "{}", it.text());

record_span(it.span);

}

tt::LitKind::Byte

@@ -620,7 +615,7 @@ fn concat_expand(

TtElement::Leaf(tt::Leaf::Literal(it))

if matches!(it.kind, tt::LitKind::Integer | tt::LitKind::Float) =>

{

- format_to!(text, "-{}", it.symbol.as_str());

+ format_to!(text, "-{}", it.text());

record_span(punct.span.cover(it.span));

}

_ => {

@@ -658,26 +653,22 @@ fn concat_bytes_expand(

};

for (i, t) in tt.iter().enumerate() {

match t {

- TtElement::Leaf(tt::Leaf::Literal(tt::Literal {

- symbol: text,

- span,

- kind,

- suffix: _,

- })) => {

+ TtElement::Leaf(tt::Leaf::Literal(lit @ tt::Literal { span, kind, .. })) => {

+ let text = lit.text();

record_span(span);

match kind {

tt::LitKind::Byte => {

- if let Ok(b) = unescape_byte(text.as_str()) {

+ if let Ok(b) = unescape_byte(text) {

bytes.extend(

b.escape_ascii().filter_map(|it| char::from_u32(it as u32)),

);

}

tt::LitKind::ByteStr => {

- bytes.push_str(text.as_str());

+ bytes.push_str(text);

}

tt::LitKind::ByteStrRaw(_) => {

- bytes.extend(text.as_str().escape_debug());

+ bytes.extend(text.escape_debug());

}

_ => {

err.get_or_insert(ExpandError::other(span, "unexpected token"));

@@ -706,12 +697,7 @@ fn concat_bytes_expand(

ExpandResult {

value: tt::TopSubtree::invisible_from_leaves(

span,

- [tt::Leaf::Literal(tt::Literal {

- symbol: Symbol::intern(&bytes),

- span,

- kind: tt::LitKind::ByteStr,

- suffix: None,

- })],

+ [tt::Leaf::Literal(tt::Literal::new_no_suffix(&bytes, span, tt::LitKind::ByteStr))],

err,

}

@@ -725,25 +711,19 @@ fn concat_bytes_expand_subtree(

) -> Result<(), ExpandError> {

for (ti, tt) in tree_iter.enumerate() {

match tt {

- TtElement::Leaf(tt::Leaf::Literal(tt::Literal {

- symbol: text,

- span,

- kind: tt::LitKind::Byte,

- suffix: _,

- })) => {

- if let Ok(b) = unescape_byte(text.as_str()) {

+ TtElement::Leaf(tt::Leaf::Literal(

+ lit @ tt::Literal { span, kind: tt::LitKind::Byte, .. },

+ )) => {

+ if let Ok(b) = unescape_byte(lit.text()) {

bytes.extend(b.escape_ascii().filter_map(|it| char::from_u32(it as u32)));

}

record_span(span);

}

- TtElement::Leaf(tt::Leaf::Literal(tt::Literal {

- symbol: text,

- span,

- kind: tt::LitKind::Integer,

- suffix: _,

- })) => {

+ TtElement::Leaf(tt::Leaf::Literal(

+ lit @ tt::Literal { span, kind: tt::LitKind::Integer, .. },

+ )) => {

record_span(span);

- if let Ok(b) = text.as_str().parse::<u8>() {

+ if let Ok(b) = lit.text().parse::<u8>() {

bytes.extend(b.escape_ascii().filter_map(|it| char::from_u32(it as u32)));

}

@@ -792,18 +772,16 @@ fn parse_string(tt: &tt::TopSubtree) -> Result<(Symbol, Span), ExpandError> {

}

match tt {

- TtElement::Leaf(tt::Leaf::Literal(tt::Literal {

- symbol: text,

+ TtElement::Leaf(tt::Leaf::Literal(lit @ tt::Literal {

span,

kind: tt::LitKind::Str,

- suffix: _,

- })) => Ok((unescape_symbol(&text), span)),

- TtElement::Leaf(tt::Leaf::Literal(tt::Literal {

- symbol: text,

+ ..

+ })) => Ok((Symbol::intern(&unescape_str(lit.text())), span)),

+ TtElement::Leaf(tt::Leaf::Literal(lit @ tt::Literal {

span,

kind: tt::LitKind::StrRaw(_),

- suffix: _,

- })) => Ok((text.clone(), span)),

+ ..

+ })) => Ok((Symbol::intern(lit.text()), span)),

TtElement::Leaf(l) => Err(*l.span()),

TtElement::Subtree(tt, _) => Err(tt.delimiter.open.cover(tt.delimiter.close)),

}

@@ -855,10 +833,10 @@ fn include_bytes_expand(

let res = tt::TopSubtree::invisible_from_leaves(

span,

[tt::Leaf::Literal(tt::Literal {

- symbol: Symbol::empty(),

+ text_and_suffix: Symbol::empty(),

span,

kind: tt::LitKind::ByteStrRaw(1),

- suffix: None,

+ suffix_len: 0,

})],

);

ExpandResult::ok(res)

@@ -979,17 +957,16 @@ fn quote_expand(

)

}

-fn unescape_symbol(s: &Symbol) -> Symbol {

- if s.as_str().contains('\\') {

- let s = s.as_str();

+fn unescape_str(s: &str) -> Cow<'_, str> {

+ if s.contains('\\') {

let mut buf = String::with_capacity(s.len());

- unescape_str(s, |_, c| {

+ syntax::unescape::unescape_str(s, |_, c| {

if let Ok(c) = c {

buf.push(c)

}

});

- Symbol::intern(&buf)

+ Cow::Owned(buf)

} else {

- s.clone()

+ Cow::Borrowed(s)

}

diff --git a/crates/hir-expand/src/builtin/quote.rs b/crates/hir-expand/src/builtin/quote.rs
index 4039b6c334..51c4e22516 100644
--- a/crates/hir-expand/src/builtin/quote.rs
+++ b/crates/hir-expand/src/builtin/quote.rs

@@ -199,16 +199,16 @@ impl<T: ToTokenTree + Clone> ToTokenTree for &T {

}

impl_to_to_tokentrees! {

- span: u32 => self { crate::tt::Literal{symbol: Symbol::integer(self as _), span, kind: tt::LitKind::Integer, suffix: None } };

- span: usize => self { crate::tt::Literal{symbol: Symbol::integer(self as _), span, kind: tt::LitKind::Integer, suffix: None } };

- span: i32 => self { crate::tt::Literal{symbol: Symbol::integer(self as _), span, kind: tt::LitKind::Integer, suffix: None } };

+ span: u32 => self { crate::tt::Literal{text_and_suffix: Symbol::integer(self as _), span, kind: tt::LitKind::Integer, suffix_len: 0 } };

+ span: usize => self { crate::tt::Literal{text_and_suffix: Symbol::integer(self as _), span, kind: tt::LitKind::Integer, suffix_len: 0 } };

+ span: i32 => self { crate::tt::Literal{text_and_suffix: Symbol::integer(self as _), span, kind: tt::LitKind::Integer, suffix_len: 0 } };

span: bool => self { crate::tt::Ident{sym: if self { sym::true_ } else { sym::false_ }, span, is_raw: tt::IdentIsRaw::No } };

_span: crate::tt::Leaf => self { self };

_span: crate::tt::Literal => self { self };

_span: crate::tt::Ident => self { self };

_span: crate::tt::Punct => self { self };

- span: &str => self { crate::tt::Literal{symbol: Symbol::intern(&self.escape_default().to_smolstr()), span, kind: tt::LitKind::Str, suffix: None }};

- span: String => self { crate::tt::Literal{symbol: Symbol::intern(&self.escape_default().to_smolstr()), span, kind: tt::LitKind::Str, suffix: None }};

+ span: &str => self { crate::tt::Literal{text_and_suffix: Symbol::intern(&self.escape_default().to_smolstr()), span, kind: tt::LitKind::Str, suffix_len: 0 }};

+ span: String => self { crate::tt::Literal{text_and_suffix: Symbol::intern(&self.escape_default().to_smolstr()), span, kind: tt::LitKind::Str, suffix_len: 0 }};

span: Name => self {

let (is_raw, s) = IdentIsRaw::split_from_symbol(self.as_str());

crate::tt::Ident{sym: Symbol::intern(s), span, is_raw }

diff --git a/crates/hir-expand/src/fixup.rs b/crates/hir-expand/src/fixup.rs
index 6afc6e5d5e..92ddd7fa8b 100644
--- a/crates/hir-expand/src/fixup.rs
+++ b/crates/hir-expand/src/fixup.rs

@@ -415,7 +415,7 @@ mod tests {

// `TokenTree`s, see the last assertion in `check()`.

fn check_leaf_eq(a: &tt::Leaf, b: &tt::Leaf) -> bool {

match (a, b) {

- (tt::Leaf::Literal(a), tt::Leaf::Literal(b)) => a.symbol == b.symbol,

+ (tt::Leaf::Literal(a), tt::Leaf::Literal(b)) => a.text_and_suffix == b.text_and_suffix,

(tt::Leaf::Punct(a), tt::Leaf::Punct(b)) => a.char == b.char,

(tt::Leaf::Ident(a), tt::Leaf::Ident(b)) => a.sym == b.sym,

_ => false,

diff --git a/crates/mbe/src/benchmark.rs b/crates/mbe/src/benchmark.rs
index 6b018510be..603fee7306 100644
--- a/crates/mbe/src/benchmark.rs
+++ b/crates/mbe/src/benchmark.rs

@@ -236,12 +236,7 @@ fn invocation_fixtures(

tt::Leaf::Punct(tt::Punct { span: DUMMY, char, spacing: tt::Spacing::Alone })

}

fn make_literal(lit: &str) -> tt::Leaf {

- tt::Leaf::Literal(tt::Literal {

- span: DUMMY,

- symbol: Symbol::intern(lit),

- kind: tt::LitKind::Str,

- suffix: None,

- })

+ tt::Leaf::Literal(tt::Literal::new_no_suffix(lit, DUMMY, tt::LitKind::Str))

}

fn make_subtree(kind: tt::DelimiterKind, builder: &mut tt::TopSubtreeBuilder) {

builder.open(kind, DUMMY);

diff --git a/crates/mbe/src/expander/matcher.rs b/crates/mbe/src/expander/matcher.rs
index e845b1ab8d..8f6627a60f 100644
--- a/crates/mbe/src/expander/matcher.rs
+++ b/crates/mbe/src/expander/matcher.rs

@@ -517,7 +517,8 @@ fn match_loop_inner<'t>(

}

OpDelimited::Op(Op::Literal(lhs)) => {

if let Ok(rhs) = src.clone().expect_leaf() {

- if matches!(&rhs, tt::Leaf::Literal(it) if it.symbol == lhs.symbol) {

+ if matches!(&rhs, tt::Leaf::Literal(it) if it.text_and_suffix == lhs.text_and_suffix)

+ {

item.dot.next();

} else {

res.add_err(ExpandError::new(

@@ -953,8 +954,8 @@ fn expect_separator(iter: &mut TtIter<'_>, separator: &Separator) -> bool {

Separator::Literal(lhs) => match fork.expect_literal() {

Ok(rhs) => match rhs {

- tt::Leaf::Literal(rhs) => rhs.symbol == lhs.symbol,

- tt::Leaf::Ident(rhs) => rhs.sym == lhs.symbol,

+ tt::Leaf::Literal(rhs) => rhs.text_and_suffix == lhs.text_and_suffix,

+ tt::Leaf::Ident(rhs) => rhs.sym == lhs.text_and_suffix,

tt::Leaf::Punct(_) => false,

Err(_) => false,

diff --git a/crates/mbe/src/expander/transcriber.rs b/crates/mbe/src/expander/transcriber.rs
index 38098e2c84..e8e7928c26 100644
--- a/crates/mbe/src/expander/transcriber.rs
+++ b/crates/mbe/src/expander/transcriber.rs

@@ -222,10 +222,10 @@ fn expand_subtree(

let index =

ctx.nesting.get(ctx.nesting.len() - 1 - depth).map_or(0, |nest| nest.idx);

builder.push(tt::Leaf::Literal(tt::Literal {

- symbol: Symbol::integer(index),

+ text_and_suffix: Symbol::integer(index),

span: ctx.call_site,

kind: tt::LitKind::Integer,

- suffix: None,

+ suffix_len: 0,

}));

}

Op::Len { depth } => {

@@ -234,10 +234,10 @@ fn expand_subtree(

});

builder.push(tt::Leaf::Literal(tt::Literal {

- symbol: Symbol::integer(length),

+ text_and_suffix: Symbol::integer(length),

span: ctx.call_site,

kind: tt::LitKind::Integer,

- suffix: None,

+ suffix_len: 0,

}));

}

Op::Count { name, depth } => {

@@ -278,9 +278,9 @@ fn expand_subtree(

let res = count(binding, 0, depth.unwrap_or(0));

builder.push(tt::Leaf::Literal(tt::Literal {

- symbol: Symbol::integer(res),

+ text_and_suffix: Symbol::integer(res),

span: ctx.call_site,

- suffix: None,

+ suffix_len: 0,

kind: tt::LitKind::Integer,

}));

}

@@ -294,7 +294,7 @@ fn expand_subtree(

ConcatMetaVarExprElem::Literal(lit) => {

// FIXME: This isn't really correct wrt. escaping, but that's what rustc does and anyway

// escaping is used most of the times for characters that are invalid in identifiers.

- concatenated.push_str(lit.symbol.as_str())

+ concatenated.push_str(lit.text())

}

ConcatMetaVarExprElem::Var(var) => {

// Handling of repetitions in `${concat}` isn't fleshed out in rustc, so we currently

@@ -329,9 +329,7 @@ fn expand_subtree(

(Some(TtElement::Leaf(tt::Leaf::Ident(ident))), None) => {

ident.sym.as_str()

}

- (Some(TtElement::Leaf(tt::Leaf::Literal(lit))), None) => {

- lit.symbol.as_str()

- }

+ (Some(TtElement::Leaf(tt::Leaf::Literal(lit))), None) => lit.text(),

_ => {

if err.is_none() {

err = Some(ExpandError::binding_error(

diff --git a/crates/mbe/src/parser.rs b/crates/mbe/src/parser.rs
index cecdd43a42..796ee62d48 100644
--- a/crates/mbe/src/parser.rs
+++ b/crates/mbe/src/parser.rs

@@ -192,7 +192,7 @@ impl PartialEq for Separator {

match (self, other) {

(Ident(a), Ident(b)) => a.sym == b.sym,

- (Literal(a), Literal(b)) => a.symbol == b.symbol,

+ (Literal(a), Literal(b)) => a.text_and_suffix == b.text_and_suffix,

(Puncts(a), Puncts(b)) if a.len() == b.len() => {

let a_iter = a.iter().map(|a| a.char);

let b_iter = b.iter().map(|b| b.char);

@@ -270,7 +270,7 @@ fn next_op(

}

tt::Leaf::Literal(lit) if is_boolean_literal(&lit) => {

let kind = eat_fragment_kind(edition, src, mode)?;

- let name = lit.symbol.clone();

+ let name = lit.text_and_suffix.clone();

let id = lit.span;

Op::Var { name, kind, id }

}

@@ -364,7 +364,7 @@ fn eat_fragment_kind(

}

fn is_boolean_literal(lit: &tt::Literal) -> bool {

- matches!(lit.symbol.as_str(), "true" | "false")

+ lit.text_and_suffix == sym::true_ || lit.text_and_suffix == sym::false_

}

fn parse_repeat(src: &mut TtIter<'_>) -> Result<(Option<Separator>, RepeatKind), ParseError> {

@@ -478,11 +478,12 @@ fn parse_metavar_expr(src: &mut TtIter<'_>) -> Result<Op, ()> {

fn parse_depth(src: &mut TtIter<'_>) -> Result<usize, ()> {

if src.is_empty() {

Ok(0)

- } else if let tt::Leaf::Literal(tt::Literal { symbol: text, suffix: None, .. }) =

- src.expect_literal()?

+ } else if let tt::Leaf::Literal(lit) = src.expect_literal()?

+ && let (text, suffix) = lit.text_and_suffix()

+ && suffix.is_empty()

{

// Suffixes are not allowed.

- text.as_str().parse().map_err(|_| ())

+ text.parse().map_err(|_| ())

} else {

Err(())

}

diff --git a/crates/proc-macro-api/src/legacy_protocol/msg.rs b/crates/proc-macro-api/src/legacy_protocol/msg.rs
index ac627b9232..4146b619ec 100644
--- a/crates/proc-macro-api/src/legacy_protocol/msg.rs
+++ b/crates/proc-macro-api/src/legacy_protocol/msg.rs

@@ -172,7 +172,7 @@ impl Message for Response {}

#[cfg(test)]

mod tests {

- use intern::{Symbol, sym};

+ use intern::Symbol;

use span::{

Edition, ROOT_ERASED_FILE_AST_ID, Span, SpanAnchor, SyntaxContext, TextRange, TextSize,

};

@@ -232,16 +232,15 @@ mod tests {

}

.into(),

);

- builder.push(Leaf::Literal(Literal {

- symbol: Symbol::intern("Foo"),

- span: Span {

+ builder.push(Leaf::Literal(Literal::new_no_suffix(

+ "Foo",

+ Span {

range: TextRange::at(TextSize::new(10), TextSize::of("\"Foo\"")),

anchor,

ctx: SyntaxContext::root(Edition::CURRENT),

- kind: tt::LitKind::Str,

- suffix: None,

- }));

+ tt::LitKind::Str,

+ )));

builder.push(Leaf::Punct(Punct {

char: '@',

span: Span {

@@ -267,16 +266,16 @@ mod tests {

ctx: SyntaxContext::root(Edition::CURRENT),

);

- builder.push(Leaf::Literal(Literal {

- symbol: sym::INTEGER_0,

- span: Span {

+ builder.push(Leaf::Literal(Literal::new(

+ "0",

+ Span {

range: TextRange::at(TextSize::new(16), TextSize::of("0u32")),

anchor,

ctx: SyntaxContext::root(Edition::CURRENT),

- kind: tt::LitKind::Integer,

- suffix: Some(sym::u32),

- }));

+ tt::LitKind::Integer,

+ "u32",

+ )));

builder.close(Span {

range: TextRange::at(TextSize::new(20), TextSize::of(']')),

anchor,

diff --git a/crates/proc-macro-api/src/legacy_protocol/msg/flat.rs b/crates/proc-macro-api/src/legacy_protocol/msg/flat.rs
index ed4b7aff76..cd8944aa61 100644
--- a/crates/proc-macro-api/src/legacy_protocol/msg/flat.rs
+++ b/crates/proc-macro-api/src/legacy_protocol/msg/flat.rs

@@ -512,12 +512,14 @@ impl<'a, T: SpanTransformer<Span = span::Span>> Writer<'a, '_, T, tt::iter::TtIt

let idx = self.literal.len() as u32;

let id = self.token_id_of(lit.span);

let (text, suffix) = if self.version >= EXTENDED_LEAF_DATA {

+ let (text, suffix) = lit.text_and_suffix();

(

- self.intern_owned(lit.symbol.as_str().to_owned()),

- lit.suffix

- .as_ref()

- .map(|s| self.intern_owned(s.as_str().to_owned()))

- .unwrap_or(!0),

+ self.intern_owned(text.to_owned()),

+ if suffix.is_empty() {

+ !0

+ } else {

+ self.intern_owned(suffix.to_owned())

+ },

)

} else {

(self.intern_owned(format!("{lit}")), !0)

@@ -774,10 +776,10 @@ impl<T: SpanTransformer<Span = span::Span>> Reader<'_, T> {

let span = read_span(repr.id);

s.push(

tt::Leaf::Literal(if self.version >= EXTENDED_LEAF_DATA {

- tt::Literal {

- symbol: Symbol::intern(text),

+ tt::Literal::new(

+ text,

span,

- kind: match u16::to_le_bytes(repr.kind) {

+ match u16::to_le_bytes(repr.kind) {

[0, _] => Err(()),

[1, _] => Byte,

[2, _] => Char,

@@ -791,14 +793,12 @@ impl<T: SpanTransformer<Span = span::Span>> Reader<'_, T> {

[10, r] => CStrRaw(r),

_ => unreachable!(),

- suffix: if repr.suffix != !0 {

- Some(Symbol::intern(

- self.text[repr.suffix as usize].as_str(),

- ))

+ if repr.suffix != !0 {

+ self.text[repr.suffix as usize].as_str()

} else {

- None

+ ""

- }

+ )

} else {

tt::token_to_literal(text, span)

})

diff --git a/crates/syntax-bridge/src/lib.rs b/crates/syntax-bridge/src/lib.rs
index 79b51a816e..ce238eb932 100644
--- a/crates/syntax-bridge/src/lib.rs
+++ b/crates/syntax-bridge/src/lib.rs

@@ -442,7 +442,7 @@ fn convert_doc_comment(

text = &text[0..text.len() - 2];

}

let (text, kind) = desugar_doc_comment_text(text, mode);

- let lit = tt::Literal { symbol: text, span, kind, suffix: None };

+ let lit = tt::Literal { text_and_suffix: text, span, kind, suffix_len: 0 };

tt::Leaf::from(lit)

};

@@ -867,12 +867,9 @@ impl TtTreeSink<'_> {

/// This occurs when a float literal is used as a field access.

fn float_split(&mut self, has_pseudo_dot: bool) {

let (text, span) = match self.cursor.token_tree() {

- Some(tt::TokenTree::Leaf(tt::Leaf::Literal(tt::Literal {

- symbol: text,

- span,

- kind: tt::LitKind::Float,

- suffix: _,

- }))) => (text.as_str(), *span),

+ Some(tt::TokenTree::Leaf(tt::Leaf::Literal(

+ lit @ tt::Literal { span, kind: tt::LitKind::Float, .. },

+ ))) => (lit.text(), *span),

tt => unreachable!("{tt:?}"),

};

// FIXME: Span splitting

diff --git a/crates/syntax-bridge/src/to_parser_input.rs b/crates/syntax-bridge/src/to_parser_input.rs
index 6883f71307..851a4af864 100644
--- a/crates/syntax-bridge/src/to_parser_input.rs
+++ b/crates/syntax-bridge/src/to_parser_input.rs

@@ -52,7 +52,7 @@ pub fn to_parser_input(

};

res.push(kind, ctx_edition(lit.span.ctx));

- if kind == FLOAT_NUMBER && !lit.symbol.as_str().ends_with('.') {

+ if kind == FLOAT_NUMBER && !lit.text().ends_with('.') {

// Tag the token as joint if it is float with a fractional part

// we use this jointness to inform the parser about what token split

// event to emit when we encounter a float literal in a field access

diff --git a/crates/test-fixture/src/lib.rs b/crates/test-fixture/src/lib.rs
index 487171c3e3..b9c389c769 100644
--- a/crates/test-fixture/src/lib.rs
+++ b/crates/test-fixture/src/lib.rs

@@ -920,7 +920,8 @@ impl ProcMacroExpander for ShortenProcMacroExpander {

Leaf::Literal(it) => {

// XXX Currently replaces any literals with an empty string, but supporting

// "shortening" other literals would be nice.

- it.symbol = Symbol::empty();

+ it.text_and_suffix = Symbol::empty();

+ it.suffix_len = 0;

}

Leaf::Punct(_) => {}

Leaf::Ident(it) => {

@@ -954,7 +955,7 @@ impl ProcMacroExpander for Issue17479ProcMacroExpander {

let Some(TtElement::Leaf(tt::Leaf::Literal(lit))) = iter.next() else {

return Err(ProcMacroExpansionError::Panic("incorrect Input".into()));

};

- let symbol = &lit.symbol;

+ let symbol = Symbol::intern(lit.text());

let span = lit.span;

Ok(quote! { span =>

#symbol()

diff --git a/crates/tt/src/lib.rs b/crates/tt/src/lib.rs
index 97e0b34ad7..91fcec9327 100644
--- a/crates/tt/src/lib.rs
+++ b/crates/tt/src/lib.rs

@@ -18,6 +18,7 @@ pub mod iter;

use std::fmt;

+use arrayvec::ArrayString;

use buffer::Cursor;

use intern::Symbol;

use stdx::{impl_from, itertools::Itertools as _};

@@ -617,17 +618,56 @@ pub enum DelimiterKind {

#[derive(Debug, Clone, PartialEq, Eq, Hash)]

pub struct Literal {

- // escaped

- pub symbol: Symbol,

+ /// Escaped, text then suffix concatenated.

+ pub text_and_suffix: Symbol,

pub span: Span,

pub kind: LitKind,

- pub suffix: Option<Symbol>,

+ pub suffix_len: u8,

}

-pub fn token_to_literal(text: &str, span: Span) -> Literal

-where

- Span: Copy,

+impl Literal {

+ #[inline]

+ pub fn text_and_suffix(&self) -> (&str, &str) {

+ let text_and_suffix = self.text_and_suffix.as_str();

+ text_and_suffix.split_at(text_and_suffix.len() - usize::from(self.suffix_len))

+ }

+ #[inline]

+ pub fn text(&self) -> &str {

+ self.text_and_suffix().0

+ }

+ #[inline]

+ pub fn suffix(&self) -> &str {

+ self.text_and_suffix().1

+ }

+ pub fn new(text: &str, span: Span, kind: LitKind, suffix: &str) -> Self {

+ const MAX_INLINE_CAPACITY: usize = 30;

+ let text_and_suffix = if suffix.is_empty() {

+ Symbol::intern(text)

+ } else if (text.len() + suffix.len()) < MAX_INLINE_CAPACITY {

+ let mut text_and_suffix = ArrayString::<MAX_INLINE_CAPACITY>::new();

+ text_and_suffix.push_str(text);

+ text_and_suffix.push_str(suffix);

+ Symbol::intern(&text_and_suffix)

+ } else {

+ let mut text_and_suffix = String::with_capacity(text.len() + suffix.len());

+ text_and_suffix.push_str(text);

+ text_and_suffix.push_str(suffix);

+ Symbol::intern(&text_and_suffix)

+ };

+ Self { text_and_suffix, span, kind, suffix_len: suffix.len().try_into().unwrap() }

+ }

+ #[inline]

+ pub fn new_no_suffix(text: &str, span: Span, kind: LitKind) -> Self {

+ Self { text_and_suffix: Symbol::intern(text), span, kind, suffix_len: 0 }

+ }

+pub fn token_to_literal(text: &str, span: Span) -> Literal {

use rustc_lexer::LiteralKind;

let token = rustc_lexer::tokenize(text, rustc_lexer::FrontmatterAllowed::No).next_tuple();

@@ -636,12 +676,7 @@ where

},)) = token

else {

- return Literal {

- span,

- symbol: Symbol::intern(text),

- kind: LitKind::Err(()),

- suffix: None,

- };

+ return Literal::new_no_suffix(text, span, LitKind::Err(()));

};

let (kind, start_offset, end_offset) = match kind {

@@ -672,20 +707,15 @@ where

let (lit, suffix) = text.split_at(suffix_start as usize);

let lit = &lit[start_offset..lit.len() - end_offset];

let suffix = match suffix {

- "" | "_" => None,

+ "" | "_" => "",

// ill-suffixed literals

_ if !matches!(kind, LitKind::Integer | LitKind::Float | LitKind::Err(_)) => {

- return Literal {

- span,

- symbol: Symbol::intern(text),

- kind: LitKind::Err(()),

- suffix: None,

- };

+ return Literal::new_no_suffix(text, span, LitKind::Err(()));

}

- suffix => Some(Symbol::intern(suffix)),

+ suffix => suffix,

};

- Literal { span, symbol: Symbol::intern(lit), kind, suffix }

+ Literal::new(lit, span, kind, suffix)

}

#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]

@@ -805,15 +835,8 @@ fn print_debug_token(f: &mut fmt::Formatter<'_>, level: usize, tt: TtElement<'_>

match tt {

TtElement::Leaf(leaf) => match leaf {

Leaf::Literal(lit) => {

- write!(

- f,

- "{}LITERAL {:?} {}{} {:#?}",

- align,

- lit.kind,

- lit.symbol,

- lit.suffix.as_ref().map(|it| it.as_str()).unwrap_or(""),

- lit.span

- )?;

+ let (text, suffix) = lit.text_and_suffix();

+ write!(f, "{}LITERAL {:?} {}{} {:#?}", align, lit.kind, text, suffix, lit.span)?;

}

Leaf::Punct(punct) => {

write!(

@@ -875,44 +898,28 @@ impl fmt::Display for Ident {

impl fmt::Display for Literal {

fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {

+ let (text, suffix) = self.text_and_suffix();

match self.kind {

- LitKind::Byte => write!(f, "b'{}'", self.symbol),

- LitKind::Char => write!(f, "'{}'", self.symbol),

- LitKind::Integer | LitKind::Float | LitKind::Err(_) => write!(f, "{}", self.symbol),

- LitKind::Str => write!(f, "\"{}\"", self.symbol),

- LitKind::ByteStr => write!(f, "b\"{}\"", self.symbol),

- LitKind::CStr => write!(f, "c\"{}\"", self.symbol),

+ LitKind::Byte => write!(f, "b'{}'", text),

+ LitKind::Char => write!(f, "'{}'", text),

+ LitKind::Integer | LitKind::Float | LitKind::Err(_) => write!(f, "{}", text),

+ LitKind::Str => write!(f, "\"{}\"", text),

+ LitKind::ByteStr => write!(f, "b\"{}\"", text),

+ LitKind::CStr => write!(f, "c\"{}\"", text),

LitKind::StrRaw(num_of_hashes) => {

let num_of_hashes = num_of_hashes as usize;

- write!(

- f,

- r#"r{0:#<num_of_hashes$}"{text}"{0:#<num_of_hashes$}"#,

- "",

- text = self.symbol

- )

+ write!(f, r#"r{0:#<num_of_hashes$}"{text}"{0:#<num_of_hashes$}"#, "", text = text)

}

LitKind::ByteStrRaw(num_of_hashes) => {

let num_of_hashes = num_of_hashes as usize;

- write!(

- f,

- r#"br{0:#<num_of_hashes$}"{text}"{0:#<num_of_hashes$}"#,

- "",

- text = self.symbol

- )

+ write!(f, r#"br{0:#<num_of_hashes$}"{text}"{0:#<num_of_hashes$}"#, "", text = text)

}

LitKind::CStrRaw(num_of_hashes) => {

let num_of_hashes = num_of_hashes as usize;

- write!(

- f,

- r#"cr{0:#<num_of_hashes$}"{text}"{0:#<num_of_hashes$}"#,

- "",

- text = self.symbol

- )

+ write!(f, r#"cr{0:#<num_of_hashes$}"{text}"{0:#<num_of_hashes$}"#, "", text = text)

}

}?;

- if let Some(suffix) = &self.suffix {

- write!(f, "{suffix}")?;

- }

+ write!(f, "{suffix}")?;

Ok(())

}

@@ -967,7 +974,7 @@ impl TopSubtree {

match tt {

TokenTree::Leaf(it) => {

let s = match it {

- Leaf::Literal(it) => it.symbol.to_string(),

+ Leaf::Literal(it) => it.text().to_owned(),

Leaf::Punct(it) => it.char.to_string(),

Leaf::Ident(it) => format!("{}{}", it.is_raw.as_str(), it.sym),

};