Unnamed repository; edit this file 'description' to name the repository.
Diffstat (limited to 'crates/proc-macro-api/src/msg/flat.rs')
| -rw-r--r-- | crates/proc-macro-api/src/msg/flat.rs | 185 |
1 files changed, 160 insertions, 25 deletions
diff --git a/crates/proc-macro-api/src/msg/flat.rs b/crates/proc-macro-api/src/msg/flat.rs index 11fd7596f2..a8661f59b2 100644 --- a/crates/proc-macro-api/src/msg/flat.rs +++ b/crates/proc-macro-api/src/msg/flat.rs @@ -37,13 +37,14 @@ use std::collections::VecDeque; +use intern::Symbol; use la_arena::RawIdx; use rustc_hash::FxHashMap; use serde::{Deserialize, Serialize}; -use span::{ErasedFileAstId, FileId, Span, SpanAnchor, SyntaxContextId}; +use span::{EditionedFileId, ErasedFileAstId, Span, SpanAnchor, SyntaxContextId}; use text_size::TextRange; -use crate::msg::ENCODE_CLOSE_SPAN_VERSION; +use crate::msg::{ENCODE_CLOSE_SPAN_VERSION, EXTENDED_LEAF_DATA}; pub type SpanDataIndexMap = indexmap::IndexSet<Span, std::hash::BuildHasherDefault<rustc_hash::FxHasher>>; @@ -52,7 +53,7 @@ pub fn serialize_span_data_index_map(map: &SpanDataIndexMap) -> Vec<u32> { map.iter() .flat_map(|span| { [ - span.anchor.file_id.index(), + span.anchor.file_id.as_u32(), span.anchor.ast_id.into_raw().into_u32(), span.range.start().into(), span.range.end().into(), @@ -69,7 +70,7 @@ pub fn deserialize_span_data_index_map(map: &[u32]) -> SpanDataIndexMap { let &[file_id, ast_id, start, end, e] = span else { unreachable!() }; Span { anchor: SpanAnchor { - file_id: FileId::from_raw(file_id), + file_id: EditionedFileId::from_raw(file_id), ast_id: ErasedFileAstId::from_raw(RawIdx::from_u32(ast_id)), }, range: TextRange::new(start.into(), end.into()), @@ -108,6 +109,8 @@ struct SubtreeRepr { struct LiteralRepr { id: TokenId, text: u32, + suffix: u32, + kind: u16, } struct PunctRepr { @@ -119,6 +122,7 @@ struct PunctRepr { struct IdentRepr { id: TokenId, text: u32, + is_raw: bool, } impl FlatTree { @@ -138,6 +142,7 @@ impl FlatTree { ident: Vec::new(), token_tree: Vec::new(), text: Vec::new(), + version, }; w.write(subtree); @@ -147,9 +152,17 @@ impl FlatTree { } else { write_vec(w.subtree, SubtreeRepr::write) }, - literal: write_vec(w.literal, LiteralRepr::write), + literal: if version >= EXTENDED_LEAF_DATA { + write_vec(w.literal, LiteralRepr::write_with_kind) + } else { + write_vec(w.literal, LiteralRepr::write) + }, punct: write_vec(w.punct, PunctRepr::write), - ident: write_vec(w.ident, IdentRepr::write), + ident: if version >= EXTENDED_LEAF_DATA { + write_vec(w.ident, IdentRepr::write_with_rawness) + } else { + write_vec(w.ident, IdentRepr::write) + }, token_tree: w.token_tree, text: w.text, } @@ -167,6 +180,7 @@ impl FlatTree { ident: Vec::new(), token_tree: Vec::new(), text: Vec::new(), + version, }; w.write(subtree); @@ -176,9 +190,17 @@ impl FlatTree { } else { write_vec(w.subtree, SubtreeRepr::write) }, - literal: write_vec(w.literal, LiteralRepr::write), + literal: if version >= EXTENDED_LEAF_DATA { + write_vec(w.literal, LiteralRepr::write_with_kind) + } else { + write_vec(w.literal, LiteralRepr::write) + }, punct: write_vec(w.punct, PunctRepr::write), - ident: write_vec(w.ident, IdentRepr::write), + ident: if version >= EXTENDED_LEAF_DATA { + write_vec(w.ident, IdentRepr::write_with_rawness) + } else { + write_vec(w.ident, IdentRepr::write) + }, token_tree: w.token_tree, text: w.text, } @@ -195,12 +217,21 @@ impl FlatTree { } else { read_vec(self.subtree, SubtreeRepr::read) }, - literal: read_vec(self.literal, LiteralRepr::read), + literal: if version >= EXTENDED_LEAF_DATA { + read_vec(self.literal, LiteralRepr::read_with_kind) + } else { + read_vec(self.literal, LiteralRepr::read) + }, punct: read_vec(self.punct, PunctRepr::read), - ident: read_vec(self.ident, IdentRepr::read), + ident: if version >= EXTENDED_LEAF_DATA { + read_vec(self.ident, IdentRepr::read_with_rawness) + } else { + read_vec(self.ident, IdentRepr::read) + }, token_tree: self.token_tree, text: self.text, span_data_table, + version, } .read() } @@ -212,12 +243,21 @@ impl FlatTree { } else { read_vec(self.subtree, SubtreeRepr::read) }, - literal: read_vec(self.literal, LiteralRepr::read), + literal: if version >= EXTENDED_LEAF_DATA { + read_vec(self.literal, LiteralRepr::read_with_kind) + } else { + read_vec(self.literal, LiteralRepr::read) + }, punct: read_vec(self.punct, PunctRepr::read), - ident: read_vec(self.ident, IdentRepr::read), + ident: if version >= EXTENDED_LEAF_DATA { + read_vec(self.ident, IdentRepr::read_with_rawness) + } else { + read_vec(self.ident, IdentRepr::read) + }, token_tree: self.token_tree, text: self.text, span_data_table: &(), + version, } .read() } @@ -280,14 +320,20 @@ impl LiteralRepr { [self.id.0, self.text] } fn read([id, text]: [u32; 2]) -> LiteralRepr { - LiteralRepr { id: TokenId(id), text } + LiteralRepr { id: TokenId(id), text, kind: 0, suffix: !0 } + } + fn write_with_kind(self) -> [u32; 4] { + [self.id.0, self.text, self.kind as u32, self.suffix] + } + fn read_with_kind([id, text, kind, suffix]: [u32; 4]) -> LiteralRepr { + LiteralRepr { id: TokenId(id), text, kind: kind as u16, suffix } } } impl PunctRepr { fn write(self) -> [u32; 3] { let spacing = match self.spacing { - tt::Spacing::Alone => 0, + tt::Spacing::Alone | tt::Spacing::JointHidden => 0, tt::Spacing::Joint => 1, }; [self.id.0, self.char as u32, spacing] @@ -307,7 +353,13 @@ impl IdentRepr { [self.id.0, self.text] } fn read(data: [u32; 2]) -> IdentRepr { - IdentRepr { id: TokenId(data[0]), text: data[1] } + IdentRepr { id: TokenId(data[0]), text: data[1], is_raw: false } + } + fn write_with_rawness(self) -> [u32; 3] { + [self.id.0, self.text, self.is_raw as u32] + } + fn read_with_rawness([id, text, is_raw]: [u32; 3]) -> IdentRepr { + IdentRepr { id: TokenId(id), text, is_raw: is_raw == 1 } } } @@ -339,8 +391,9 @@ impl InternableSpan for Span { struct Writer<'a, 'span, S: InternableSpan> { work: VecDeque<(usize, &'a tt::Subtree<S>)>, - string_table: FxHashMap<&'a str, u32>, + string_table: FxHashMap<std::borrow::Cow<'a, str>, u32>, span_data_table: &'span mut S::Table, + version: u32, subtree: Vec<SubtreeRepr>, literal: Vec<LiteralRepr>, @@ -378,9 +431,33 @@ impl<'a, 'span, S: InternableSpan> Writer<'a, 'span, S> { tt::TokenTree::Leaf(leaf) => match leaf { tt::Leaf::Literal(lit) => { let idx = self.literal.len() as u32; - let text = self.intern(&lit.text); let id = self.token_id_of(lit.span); - self.literal.push(LiteralRepr { id, text }); + let (text, suffix) = if self.version >= EXTENDED_LEAF_DATA { + ( + self.intern(lit.symbol.as_str()), + lit.suffix.as_ref().map(|s| self.intern(s.as_str())).unwrap_or(!0), + ) + } else { + (self.intern_owned(format!("{lit}")), !0) + }; + self.literal.push(LiteralRepr { + id, + text, + kind: u16::from_le_bytes(match lit.kind { + tt::LitKind::Err(_) => [0, 0], + tt::LitKind::Byte => [1, 0], + tt::LitKind::Char => [2, 0], + tt::LitKind::Integer => [3, 0], + tt::LitKind::Float => [4, 0], + tt::LitKind::Str => [5, 0], + tt::LitKind::StrRaw(r) => [6, r], + tt::LitKind::ByteStr => [7, 0], + tt::LitKind::ByteStrRaw(r) => [8, r], + tt::LitKind::CStr => [9, 0], + tt::LitKind::CStrRaw(r) => [10, r], + }), + suffix, + }); idx << 2 | 0b01 } tt::Leaf::Punct(punct) => { @@ -391,9 +468,15 @@ impl<'a, 'span, S: InternableSpan> Writer<'a, 'span, S> { } tt::Leaf::Ident(ident) => { let idx = self.ident.len() as u32; - let text = self.intern(&ident.text); let id = self.token_id_of(ident.span); - self.ident.push(IdentRepr { id, text }); + let text = if self.version >= EXTENDED_LEAF_DATA { + self.intern(ident.sym.as_str()) + } else if ident.is_raw.yes() { + self.intern_owned(format!("r#{}", ident.sym.as_str(),)) + } else { + self.intern(ident.sym.as_str()) + }; + self.ident.push(IdentRepr { id, text, is_raw: ident.is_raw.yes() }); idx << 2 | 0b11 } }, @@ -415,15 +498,25 @@ impl<'a, 'span, S: InternableSpan> Writer<'a, 'span, S> { pub(crate) fn intern(&mut self, text: &'a str) -> u32 { let table = &mut self.text; - *self.string_table.entry(text).or_insert_with(|| { + *self.string_table.entry(text.into()).or_insert_with(|| { let idx = table.len(); table.push(text.to_owned()); idx as u32 }) } + + pub(crate) fn intern_owned(&mut self, text: String) -> u32 { + let table = &mut self.text; + *self.string_table.entry(text.clone().into()).or_insert_with(|| { + let idx = table.len(); + table.push(text); + idx as u32 + }) + } } struct Reader<'span, S: InternableSpan> { + version: u32, subtree: Vec<SubtreeRepr>, literal: Vec<LiteralRepr>, punct: Vec<PunctRepr>, @@ -457,10 +550,38 @@ impl<'span, S: InternableSpan> Reader<'span, S> { // that this unwrap doesn't fire. 0b00 => res[idx].take().unwrap().into(), 0b01 => { + use tt::LitKind::*; let repr = &self.literal[idx]; - tt::Leaf::Literal(tt::Literal { - text: self.text[repr.text as usize].as_str().into(), - span: read_span(repr.id), + let text = self.text[repr.text as usize].as_str(); + let span = read_span(repr.id); + tt::Leaf::Literal(if self.version >= EXTENDED_LEAF_DATA { + tt::Literal { + symbol: Symbol::intern(text), + span, + kind: match u16::to_le_bytes(repr.kind) { + [0, _] => Err(()), + [1, _] => Byte, + [2, _] => Char, + [3, _] => Integer, + [4, _] => Float, + [5, _] => Str, + [6, r] => StrRaw(r), + [7, _] => ByteStr, + [8, r] => ByteStrRaw(r), + [9, _] => CStr, + [10, r] => CStrRaw(r), + _ => unreachable!(), + }, + suffix: if repr.suffix != !0 { + Some(Symbol::intern( + self.text[repr.suffix as usize].as_str(), + )) + } else { + None + }, + } + } else { + tt::token_to_literal(text, span) }) .into() } @@ -475,9 +596,23 @@ impl<'span, S: InternableSpan> Reader<'span, S> { } 0b11 => { let repr = &self.ident[idx]; + let text = self.text[repr.text as usize].as_str(); + let (is_raw, text) = if self.version >= EXTENDED_LEAF_DATA { + ( + if repr.is_raw { + tt::IdentIsRaw::Yes + } else { + tt::IdentIsRaw::No + }, + text, + ) + } else { + tt::IdentIsRaw::split_from_symbol(text) + }; tt::Leaf::Ident(tt::Ident { - text: self.text[repr.text as usize].as_str().into(), + sym: Symbol::intern(text), span: read_span(repr.id), + is_raw, }) .into() } |