Unnamed repository; edit this file 'description' to name the repository.
Merge pull request #21249 from Shourya742/2025-11-27-bidirectional-protocol
Add bidirectional messaging proc-macro-srv
25 files changed, 903 insertions, 136 deletions
diff --git a/Cargo.lock b/Cargo.lock index a2eb8b1397..fb76f51cd3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1877,6 +1877,7 @@ name = "proc-macro-srv-cli" version = "0.0.0" dependencies = [ "clap", + "crossbeam-channel", "postcard", "proc-macro-api", "proc-macro-srv", diff --git a/crates/hir-def/src/macro_expansion_tests/mod.rs b/crates/hir-def/src/macro_expansion_tests/mod.rs index 59bd9474a9..5cb271b01d 100644 --- a/crates/hir-def/src/macro_expansion_tests/mod.rs +++ b/crates/hir-def/src/macro_expansion_tests/mod.rs @@ -16,7 +16,7 @@ mod proc_macros; use std::{any::TypeId, iter, ops::Range, sync}; -use base_db::RootQueryDb; +use base_db::{RootQueryDb, SourceDatabase}; use expect_test::Expect; use hir_expand::{ AstId, ExpansionInfo, InFile, MacroCallId, MacroCallKind, MacroKind, @@ -378,6 +378,7 @@ struct IdentityWhenValidProcMacroExpander; impl ProcMacroExpander for IdentityWhenValidProcMacroExpander { fn expand( &self, + _: &dyn SourceDatabase, subtree: &TopSubtree, _: Option<&TopSubtree>, _: &base_db::Env, diff --git a/crates/hir-expand/src/proc_macro.rs b/crates/hir-expand/src/proc_macro.rs index f97d721dfa..d2614aa5f1 100644 --- a/crates/hir-expand/src/proc_macro.rs +++ b/crates/hir-expand/src/proc_macro.rs @@ -4,7 +4,7 @@ use core::fmt; use std::any::Any; use std::{panic::RefUnwindSafe, sync}; -use base_db::{Crate, CrateBuilderId, CratesIdMap, Env, ProcMacroLoadingError}; +use base_db::{Crate, CrateBuilderId, CratesIdMap, Env, ProcMacroLoadingError, SourceDatabase}; use intern::Symbol; use rustc_hash::FxHashMap; use span::Span; @@ -25,6 +25,7 @@ pub trait ProcMacroExpander: fmt::Debug + Send + Sync + RefUnwindSafe + Any { /// [`ProcMacroKind::Attr`]), environment variables, and span information. fn expand( &self, + db: &dyn SourceDatabase, subtree: &tt::TopSubtree, attrs: Option<&tt::TopSubtree>, env: &Env, @@ -309,6 +310,7 @@ impl CustomProcMacroExpander { let current_dir = calling_crate.data(db).proc_macro_cwd.to_string(); match proc_macro.expander.expand( + db, tt, attr_arg, env, diff --git a/crates/load-cargo/src/lib.rs b/crates/load-cargo/src/lib.rs index 28fbfecfde..a7b22b0d6a 100644 --- a/crates/load-cargo/src/lib.rs +++ b/crates/load-cargo/src/lib.rs @@ -17,15 +17,23 @@ use hir_expand::proc_macro::{ }; use ide_db::{ ChangeWithProcMacros, FxHashMap, RootDatabase, - base_db::{CrateGraphBuilder, Env, ProcMacroLoadingError, SourceRoot, SourceRootId}, + base_db::{ + CrateGraphBuilder, Env, ProcMacroLoadingError, SourceDatabase, SourceRoot, SourceRootId, + }, prime_caches, }; use itertools::Itertools; -use proc_macro_api::{MacroDylib, ProcMacroClient}; +use proc_macro_api::{ + MacroDylib, ProcMacroClient, + bidirectional_protocol::{ + msg::{SubRequest, SubResponse}, + reject_subrequests, + }, +}; use project_model::{CargoConfig, PackageRoot, ProjectManifest, ProjectWorkspace}; use span::Span; use vfs::{ - AbsPath, AbsPathBuf, VfsPath, + AbsPath, AbsPathBuf, FileId, VfsPath, file_set::FileSetConfig, loader::{Handle, LoadingProgress}, }; @@ -425,7 +433,7 @@ pub fn load_proc_macro( ) -> ProcMacroLoadResult { let res: Result<Vec<_>, _> = (|| { let dylib = MacroDylib::new(path.to_path_buf()); - let vec = server.load_dylib(dylib).map_err(|e| { + let vec = server.load_dylib(dylib, Some(&mut reject_subrequests)).map_err(|e| { ProcMacroLoadingError::ProcMacroSrvError(format!("{e}").into_boxed_str()) })?; if vec.is_empty() { @@ -522,6 +530,7 @@ struct Expander(proc_macro_api::ProcMacro); impl ProcMacroExpander for Expander { fn expand( &self, + db: &dyn SourceDatabase, subtree: &tt::TopSubtree<Span>, attrs: Option<&tt::TopSubtree<Span>>, env: &Env, @@ -530,6 +539,14 @@ impl ProcMacroExpander for Expander { mixed_site: Span, current_dir: String, ) -> Result<tt::TopSubtree<Span>, ProcMacroExpansionError> { + let mut cb = |req| match req { + SubRequest::SourceText { file_id, start, end } => { + let file = FileId::from_raw(file_id); + let text = db.file_text(file).text(db); + let slice = text.get(start as usize..end as usize).map(ToOwned::to_owned); + Ok(SubResponse::SourceTextResult { text: slice }) + } + }; match self.0.expand( subtree.view(), attrs.map(|attrs| attrs.view()), @@ -538,6 +555,7 @@ impl ProcMacroExpander for Expander { call_site, mixed_site, current_dir, + Some(&mut cb), ) { Ok(Ok(subtree)) => Ok(subtree), Ok(Err(err)) => Err(ProcMacroExpansionError::Panic(err)), diff --git a/crates/proc-macro-api/src/bidirectional_protocol.rs b/crates/proc-macro-api/src/bidirectional_protocol.rs new file mode 100644 index 0000000000..e919ff48fe --- /dev/null +++ b/crates/proc-macro-api/src/bidirectional_protocol.rs @@ -0,0 +1,228 @@ +//! Bidirectional protocol methods + +use std::{ + io::{self, BufRead, Write}, + sync::Arc, +}; + +use paths::AbsPath; +use span::Span; + +use crate::{ + Codec, ProcMacro, ProcMacroKind, ServerError, + bidirectional_protocol::msg::{ + BidirectionalMessage, ExpandMacro, ExpandMacroData, ExpnGlobals, Request, Response, + SubRequest, SubResponse, + }, + legacy_protocol::{ + SpanMode, + msg::{ + FlatTree, ServerConfig, SpanDataIndexMap, deserialize_span_data_index_map, + serialize_span_data_index_map, + }, + }, + process::ProcMacroServerProcess, + transport::codec::postcard::PostcardProtocol, + version, +}; + +pub mod msg; + +pub type SubCallback<'a> = &'a mut dyn FnMut(SubRequest) -> Result<SubResponse, ServerError>; + +pub fn run_conversation<C: Codec>( + writer: &mut dyn Write, + reader: &mut dyn BufRead, + buf: &mut C::Buf, + msg: BidirectionalMessage, + callback: SubCallback<'_>, +) -> Result<BidirectionalMessage, ServerError> { + let encoded = C::encode(&msg).map_err(wrap_encode)?; + C::write(writer, &encoded).map_err(wrap_io("failed to write initial request"))?; + + loop { + let maybe_buf = C::read(reader, buf).map_err(wrap_io("failed to read message"))?; + let Some(b) = maybe_buf else { + return Err(ServerError { + message: "proc-macro server closed the stream".into(), + io: Some(Arc::new(io::Error::new(io::ErrorKind::UnexpectedEof, "closed"))), + }); + }; + + let msg: BidirectionalMessage = C::decode(b).map_err(wrap_decode)?; + + match msg { + BidirectionalMessage::Response(response) => { + return Ok(BidirectionalMessage::Response(response)); + } + BidirectionalMessage::SubRequest(sr) => { + let resp = callback(sr)?; + let reply = BidirectionalMessage::SubResponse(resp); + let encoded = C::encode(&reply).map_err(wrap_encode)?; + C::write(writer, &encoded).map_err(wrap_io("failed to write sub-response"))?; + } + _ => { + return Err(ServerError { + message: format!("unexpected message {:?}", msg), + io: None, + }); + } + } + } +} + +fn wrap_io(msg: &'static str) -> impl Fn(io::Error) -> ServerError { + move |err| ServerError { message: msg.into(), io: Some(Arc::new(err)) } +} + +fn wrap_encode(err: io::Error) -> ServerError { + ServerError { message: "failed to encode message".into(), io: Some(Arc::new(err)) } +} + +fn wrap_decode(err: io::Error) -> ServerError { + ServerError { message: "failed to decode message".into(), io: Some(Arc::new(err)) } +} + +pub(crate) fn version_check( + srv: &ProcMacroServerProcess, + callback: SubCallback<'_>, +) -> Result<u32, ServerError> { + let request = BidirectionalMessage::Request(Request::ApiVersionCheck {}); + + let response_payload = run_request(srv, request, callback)?; + + match response_payload { + BidirectionalMessage::Response(Response::ApiVersionCheck(version)) => Ok(version), + other => { + Err(ServerError { message: format!("unexpected response: {:?}", other), io: None }) + } + } +} + +/// Enable support for rust-analyzer span mode if the server supports it. +pub(crate) fn enable_rust_analyzer_spans( + srv: &ProcMacroServerProcess, + callback: SubCallback<'_>, +) -> Result<SpanMode, ServerError> { + let request = BidirectionalMessage::Request(Request::SetConfig(ServerConfig { + span_mode: SpanMode::RustAnalyzer, + })); + + let response_payload = run_request(srv, request, callback)?; + + match response_payload { + BidirectionalMessage::Response(Response::SetConfig(ServerConfig { span_mode })) => { + Ok(span_mode) + } + _ => Err(ServerError { message: "unexpected response".to_owned(), io: None }), + } +} + +/// Finds proc-macros in a given dynamic library. +pub(crate) fn find_proc_macros( + srv: &ProcMacroServerProcess, + dylib_path: &AbsPath, + callback: SubCallback<'_>, +) -> Result<Result<Vec<(String, ProcMacroKind)>, String>, ServerError> { + let request = BidirectionalMessage::Request(Request::ListMacros { + dylib_path: dylib_path.to_path_buf().into(), + }); + + let response_payload = run_request(srv, request, callback)?; + + match response_payload { + BidirectionalMessage::Response(Response::ListMacros(it)) => Ok(it), + _ => Err(ServerError { message: "unexpected response".to_owned(), io: None }), + } +} + +pub(crate) fn expand( + proc_macro: &ProcMacro, + subtree: tt::SubtreeView<'_, Span>, + attr: Option<tt::SubtreeView<'_, Span>>, + env: Vec<(String, String)>, + def_site: Span, + call_site: Span, + mixed_site: Span, + current_dir: String, + callback: SubCallback<'_>, +) -> Result<Result<tt::TopSubtree<span::SpanData<span::SyntaxContext>>, String>, crate::ServerError> +{ + let version = proc_macro.process.version(); + let mut span_data_table = SpanDataIndexMap::default(); + let def_site = span_data_table.insert_full(def_site).0; + let call_site = span_data_table.insert_full(call_site).0; + let mixed_site = span_data_table.insert_full(mixed_site).0; + let task = BidirectionalMessage::Request(Request::ExpandMacro(Box::new(ExpandMacro { + data: ExpandMacroData { + macro_body: FlatTree::from_subtree(subtree, version, &mut span_data_table), + macro_name: proc_macro.name.to_string(), + attributes: attr + .map(|subtree| FlatTree::from_subtree(subtree, version, &mut span_data_table)), + has_global_spans: ExpnGlobals { + serialize: version >= version::HAS_GLOBAL_SPANS, + def_site, + call_site, + mixed_site, + }, + span_data_table: if proc_macro.process.rust_analyzer_spans() { + serialize_span_data_index_map(&span_data_table) + } else { + Vec::new() + }, + }, + lib: proc_macro.dylib_path.to_path_buf().into(), + env, + current_dir: Some(current_dir), + }))); + + let response_payload = run_request(&proc_macro.process, task, callback)?; + + match response_payload { + BidirectionalMessage::Response(Response::ExpandMacro(it)) => Ok(it + .map(|tree| { + let mut expanded = FlatTree::to_subtree_resolved(tree, version, &span_data_table); + if proc_macro.needs_fixup_change() { + proc_macro.change_fixup_to_match_old_server(&mut expanded); + } + expanded + }) + .map_err(|msg| msg.0)), + BidirectionalMessage::Response(Response::ExpandMacroExtended(it)) => Ok(it + .map(|resp| { + let mut expanded = FlatTree::to_subtree_resolved( + resp.tree, + version, + &deserialize_span_data_index_map(&resp.span_data_table), + ); + if proc_macro.needs_fixup_change() { + proc_macro.change_fixup_to_match_old_server(&mut expanded); + } + expanded + }) + .map_err(|msg| msg.0)), + _ => Err(ServerError { message: "unexpected response".to_owned(), io: None }), + } +} + +fn run_request( + srv: &ProcMacroServerProcess, + msg: BidirectionalMessage, + callback: SubCallback<'_>, +) -> Result<BidirectionalMessage, ServerError> { + if let Some(err) = srv.exited() { + return Err(err.clone()); + } + + match srv.use_postcard() { + true => srv.run_bidirectional::<PostcardProtocol>(msg, callback), + false => Err(ServerError { + message: "bidirectional messaging does not support JSON".to_owned(), + io: None, + }), + } +} + +pub fn reject_subrequests(req: SubRequest) -> Result<SubResponse, ServerError> { + Err(ServerError { message: format!("{req:?} sub-request not supported here"), io: None }) +} diff --git a/crates/proc-macro-api/src/bidirectional_protocol/msg.rs b/crates/proc-macro-api/src/bidirectional_protocol/msg.rs new file mode 100644 index 0000000000..cf8becd922 --- /dev/null +++ b/crates/proc-macro-api/src/bidirectional_protocol/msg.rs @@ -0,0 +1,91 @@ +//! Bidirectional protocol messages + +use paths::Utf8PathBuf; +use serde::{Deserialize, Serialize}; + +use crate::{ + ProcMacroKind, + legacy_protocol::msg::{FlatTree, Message, PanicMessage, ServerConfig}, +}; + +#[derive(Debug, Serialize, Deserialize)] +pub enum SubRequest { + SourceText { file_id: u32, start: u32, end: u32 }, +} + +#[derive(Debug, Serialize, Deserialize)] +pub enum SubResponse { + SourceTextResult { text: Option<String> }, +} + +#[derive(Debug, Serialize, Deserialize)] +pub enum BidirectionalMessage { + Request(Request), + Response(Response), + SubRequest(SubRequest), + SubResponse(SubResponse), +} + +#[derive(Debug, Serialize, Deserialize)] +pub enum Request { + ListMacros { dylib_path: Utf8PathBuf }, + ExpandMacro(Box<ExpandMacro>), + ApiVersionCheck {}, + SetConfig(ServerConfig), +} + +#[derive(Debug, Serialize, Deserialize)] +pub enum Response { + ListMacros(Result<Vec<(String, ProcMacroKind)>, String>), + ExpandMacro(Result<FlatTree, PanicMessage>), + ApiVersionCheck(u32), + SetConfig(ServerConfig), + ExpandMacroExtended(Result<ExpandMacroExtended, PanicMessage>), +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct ExpandMacro { + pub lib: Utf8PathBuf, + pub env: Vec<(String, String)>, + pub current_dir: Option<String>, + #[serde(flatten)] + pub data: ExpandMacroData, +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct ExpandMacroExtended { + pub tree: FlatTree, + pub span_data_table: Vec<u32>, +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct ExpandMacroData { + pub macro_body: FlatTree, + pub macro_name: String, + pub attributes: Option<FlatTree>, + #[serde(skip_serializing_if = "ExpnGlobals::skip_serializing_if")] + #[serde(default)] + pub has_global_spans: ExpnGlobals, + + #[serde(skip_serializing_if = "Vec::is_empty")] + #[serde(default)] + pub span_data_table: Vec<u32>, +} + +#[derive(Clone, Copy, Default, Debug, Serialize, Deserialize)] +pub struct ExpnGlobals { + #[serde(skip_serializing)] + #[serde(default)] + pub serialize: bool, + pub def_site: usize, + pub call_site: usize, + pub mixed_site: usize, +} + +impl ExpnGlobals { + fn skip_serializing_if(&self) -> bool { + !self.serialize + } +} + +impl Message for BidirectionalMessage {} diff --git a/crates/proc-macro-api/src/legacy_protocol.rs b/crates/proc-macro-api/src/legacy_protocol.rs index c2b132ddcc..0d16b60025 100644 --- a/crates/proc-macro-api/src/legacy_protocol.rs +++ b/crates/proc-macro-api/src/legacy_protocol.rs @@ -1,8 +1,6 @@ //! The initial proc-macro-srv protocol, soon to be deprecated. -pub mod json; pub mod msg; -pub mod postcard; use std::{ io::{BufRead, Write}, @@ -14,17 +12,14 @@ use span::Span; use crate::{ ProcMacro, ProcMacroKind, ServerError, - codec::Codec, - legacy_protocol::{ - json::JsonProtocol, - msg::{ - ExpandMacro, ExpandMacroData, ExpnGlobals, FlatTree, Message, Request, Response, - ServerConfig, SpanDataIndexMap, deserialize_span_data_index_map, - flat::serialize_span_data_index_map, - }, - postcard::PostcardProtocol, + legacy_protocol::msg::{ + ExpandMacro, ExpandMacroData, ExpnGlobals, FlatTree, Message, Request, Response, + ServerConfig, SpanDataIndexMap, deserialize_span_data_index_map, + flat::serialize_span_data_index_map, }, process::ProcMacroServerProcess, + transport::codec::Codec, + transport::codec::{json::JsonProtocol, postcard::PostcardProtocol}, version, }; @@ -155,9 +150,9 @@ fn send_task(srv: &ProcMacroServerProcess, req: Request) -> Result<Response, Ser } if srv.use_postcard() { - srv.send_task(send_request::<PostcardProtocol>, req) + srv.send_task::<_, _, PostcardProtocol>(send_request::<PostcardProtocol>, req) } else { - srv.send_task(send_request::<JsonProtocol>, req) + srv.send_task::<_, _, JsonProtocol>(send_request::<JsonProtocol>, req) } } diff --git a/crates/proc-macro-api/src/legacy_protocol/msg.rs b/crates/proc-macro-api/src/legacy_protocol/msg.rs index a6e228d977..0ebb0e9f93 100644 --- a/crates/proc-macro-api/src/legacy_protocol/msg.rs +++ b/crates/proc-macro-api/src/legacy_protocol/msg.rs @@ -8,7 +8,7 @@ use paths::Utf8PathBuf; use serde::de::DeserializeOwned; use serde_derive::{Deserialize, Serialize}; -use crate::{ProcMacroKind, codec::Codec}; +use crate::{Codec, ProcMacroKind}; /// Represents requests sent from the client to the proc-macro-srv. #[derive(Debug, Serialize, Deserialize)] diff --git a/crates/proc-macro-api/src/lib.rs b/crates/proc-macro-api/src/lib.rs index 85b250eddf..0ee0c3afb5 100644 --- a/crates/proc-macro-api/src/lib.rs +++ b/crates/proc-macro-api/src/lib.rs @@ -16,18 +16,18 @@ #[cfg(feature = "in-rust-tree")] extern crate rustc_driver as _; -mod codec; -mod framing; +pub mod bidirectional_protocol; pub mod legacy_protocol; mod process; +pub mod transport; use paths::{AbsPath, AbsPathBuf}; use semver::Version; use span::{ErasedFileAstId, FIXUP_ERASED_FILE_AST_ID_MARKER, Span}; use std::{fmt, io, sync::Arc, time::SystemTime}; -pub use crate::codec::Codec; -use crate::process::ProcMacroServerProcess; +pub use crate::transport::codec::Codec; +use crate::{bidirectional_protocol::SubCallback, process::ProcMacroServerProcess}; /// The versions of the server protocol pub mod version { @@ -142,9 +142,13 @@ impl ProcMacroClient { } /// Loads a proc-macro dylib into the server process returning a list of `ProcMacro`s loaded. - pub fn load_dylib(&self, dylib: MacroDylib) -> Result<Vec<ProcMacro>, ServerError> { + pub fn load_dylib( + &self, + dylib: MacroDylib, + callback: Option<SubCallback<'_>>, + ) -> Result<Vec<ProcMacro>, ServerError> { let _p = tracing::info_span!("ProcMacroServer::load_dylib").entered(); - let macros = self.process.find_proc_macros(&dylib.path)?; + let macros = self.process.find_proc_macros(&dylib.path, callback)?; let dylib_path = Arc::new(dylib.path); let dylib_last_modified = std::fs::metadata(dylib_path.as_path()) @@ -225,6 +229,7 @@ impl ProcMacro { call_site: Span, mixed_site: Span, current_dir: String, + callback: Option<SubCallback<'_>>, ) -> Result<Result<tt::TopSubtree<Span>, String>, ServerError> { let (mut subtree, mut attr) = (subtree, attr); let (mut subtree_changed, mut attr_changed); @@ -240,7 +245,7 @@ impl ProcMacro { } } - legacy_protocol::expand( + self.process.expand( self, subtree, attr, @@ -249,6 +254,7 @@ impl ProcMacro { call_site, mixed_site, current_dir, + callback, ) } } diff --git a/crates/proc-macro-api/src/process.rs b/crates/proc-macro-api/src/process.rs index d6a8d27bfc..01de8e98ff 100644 --- a/crates/proc-macro-api/src/process.rs +++ b/crates/proc-macro-api/src/process.rs @@ -9,10 +9,12 @@ use std::{ use paths::AbsPath; use semver::Version; +use span::Span; use stdx::JodChild; use crate::{ - ProcMacroKind, ServerError, + Codec, ProcMacro, ProcMacroKind, ServerError, + bidirectional_protocol::{self, SubCallback, msg::BidirectionalMessage, reject_subrequests}, legacy_protocol::{self, SpanMode}, version, }; @@ -33,6 +35,7 @@ pub(crate) struct ProcMacroServerProcess { pub(crate) enum Protocol { LegacyJson { mode: SpanMode }, LegacyPostcard { mode: SpanMode }, + BidirectionalPostcardPrototype { mode: SpanMode }, } /// Maintains the state of the proc-macro server process. @@ -62,6 +65,10 @@ impl ProcMacroServerProcess { && has_working_format_flag { &[ + ( + Some("bidirectional-postcard-prototype"), + Protocol::BidirectionalPostcardPrototype { mode: SpanMode::Id }, + ), (Some("postcard-legacy"), Protocol::LegacyPostcard { mode: SpanMode::Id }), (Some("json-legacy"), Protocol::LegacyJson { mode: SpanMode::Id }), ] @@ -84,7 +91,7 @@ impl ProcMacroServerProcess { }; let mut srv = create_srv()?; tracing::info!("sending proc-macro server version check"); - match srv.version_check() { + match srv.version_check(Some(&mut reject_subrequests)) { Ok(v) if v > version::CURRENT_API_VERSION => { #[allow(clippy::disallowed_methods)] let process_version = Command::new(process_path) @@ -102,12 +109,13 @@ impl ProcMacroServerProcess { tracing::info!("Proc-macro server version: {v}"); srv.version = v; if srv.version >= version::RUST_ANALYZER_SPAN_SUPPORT - && let Ok(new_mode) = srv.enable_rust_analyzer_spans() + && let Ok(new_mode) = + srv.enable_rust_analyzer_spans(Some(&mut reject_subrequests)) { match &mut srv.protocol { - Protocol::LegacyJson { mode } | Protocol::LegacyPostcard { mode } => { - *mode = new_mode - } + Protocol::LegacyJson { mode } + | Protocol::LegacyPostcard { mode } + | Protocol::BidirectionalPostcardPrototype { mode } => *mode = new_mode, } } tracing::info!("Proc-macro server protocol: {:?}", srv.protocol); @@ -143,22 +151,36 @@ impl ProcMacroServerProcess { match self.protocol { Protocol::LegacyJson { mode } => mode == SpanMode::RustAnalyzer, Protocol::LegacyPostcard { mode } => mode == SpanMode::RustAnalyzer, + Protocol::BidirectionalPostcardPrototype { mode } => mode == SpanMode::RustAnalyzer, } } /// Checks the API version of the running proc-macro server. - fn version_check(&self) -> Result<u32, ServerError> { + fn version_check(&self, callback: Option<SubCallback<'_>>) -> Result<u32, ServerError> { match self.protocol { - Protocol::LegacyJson { .. } => legacy_protocol::version_check(self), - Protocol::LegacyPostcard { .. } => legacy_protocol::version_check(self), + Protocol::LegacyJson { .. } | Protocol::LegacyPostcard { .. } => { + legacy_protocol::version_check(self) + } + Protocol::BidirectionalPostcardPrototype { .. } => { + let cb = callback.expect("callback required for bidirectional protocol"); + bidirectional_protocol::version_check(self, cb) + } } } /// Enable support for rust-analyzer span mode if the server supports it. - fn enable_rust_analyzer_spans(&self) -> Result<SpanMode, ServerError> { + fn enable_rust_analyzer_spans( + &self, + callback: Option<SubCallback<'_>>, + ) -> Result<SpanMode, ServerError> { match self.protocol { - Protocol::LegacyJson { .. } => legacy_protocol::enable_rust_analyzer_spans(self), - Protocol::LegacyPostcard { .. } => legacy_protocol::enable_rust_analyzer_spans(self), + Protocol::LegacyJson { .. } | Protocol::LegacyPostcard { .. } => { + legacy_protocol::enable_rust_analyzer_spans(self) + } + Protocol::BidirectionalPostcardPrototype { .. } => { + let cb = callback.expect("callback required for bidirectional protocol"); + bidirectional_protocol::enable_rust_analyzer_spans(self, cb) + } } } @@ -166,30 +188,70 @@ impl ProcMacroServerProcess { pub(crate) fn find_proc_macros( &self, dylib_path: &AbsPath, + callback: Option<SubCallback<'_>>, ) -> Result<Result<Vec<(String, ProcMacroKind)>, String>, ServerError> { match self.protocol { - Protocol::LegacyJson { .. } => legacy_protocol::find_proc_macros(self, dylib_path), - Protocol::LegacyPostcard { .. } => legacy_protocol::find_proc_macros(self, dylib_path), + Protocol::LegacyJson { .. } | Protocol::LegacyPostcard { .. } => { + legacy_protocol::find_proc_macros(self, dylib_path) + } + Protocol::BidirectionalPostcardPrototype { .. } => { + let cb = callback.expect("callback required for bidirectional protocol"); + bidirectional_protocol::find_proc_macros(self, dylib_path, cb) + } + } + } + + pub(crate) fn expand( + &self, + proc_macro: &ProcMacro, + subtree: tt::SubtreeView<'_, Span>, + attr: Option<tt::SubtreeView<'_, Span>>, + env: Vec<(String, String)>, + def_site: Span, + call_site: Span, + mixed_site: Span, + current_dir: String, + callback: Option<SubCallback<'_>>, + ) -> Result<Result<tt::TopSubtree<Span>, String>, ServerError> { + match self.protocol { + Protocol::LegacyJson { .. } | Protocol::LegacyPostcard { .. } => { + legacy_protocol::expand( + proc_macro, + subtree, + attr, + env, + def_site, + call_site, + mixed_site, + current_dir, + ) + } + Protocol::BidirectionalPostcardPrototype { .. } => bidirectional_protocol::expand( + proc_macro, + subtree, + attr, + env, + def_site, + call_site, + mixed_site, + current_dir, + callback.expect("callback required for bidirectional protocol"), + ), } } - pub(crate) fn send_task<Request, Response, Buf>( + pub(crate) fn send_task<Request, Response, C: Codec>( &self, - serialize_req: impl FnOnce( + send: impl FnOnce( &mut dyn Write, &mut dyn BufRead, Request, - &mut Buf, + &mut C::Buf, ) -> Result<Option<Response>, ServerError>, req: Request, - ) -> Result<Response, ServerError> - where - Buf: Default, - { - let state = &mut *self.state.lock().unwrap(); - let mut buf = Buf::default(); - serialize_req(&mut state.stdin, &mut state.stdout, req, &mut buf) - .and_then(|res| { + ) -> Result<Response, ServerError> { + self.with_locked_io::<C, _>(|writer, reader, buf| { + send(writer, reader, req, buf).and_then(|res| { res.ok_or_else(|| { let message = "proc-macro server did not respond with data".to_owned(); ServerError { @@ -201,33 +263,51 @@ impl ProcMacroServerProcess { } }) }) - .map_err(|e| { - if e.io.as_ref().map(|it| it.kind()) == Some(io::ErrorKind::BrokenPipe) { - match state.process.child.try_wait() { - Ok(None) | Err(_) => e, - Ok(Some(status)) => { - let mut msg = String::new(); - if !status.success() - && let Some(stderr) = state.process.child.stderr.as_mut() - { - _ = stderr.read_to_string(&mut msg); - } - let server_error = ServerError { - message: format!( - "proc-macro server exited with {status}{}{msg}", - if msg.is_empty() { "" } else { ": " } - ), - io: None, - }; - // `AssertUnwindSafe` is fine here, we already correct initialized - // server_error at this point. - self.exited.get_or_init(|| AssertUnwindSafe(server_error)).0.clone() + }) + } + + pub(crate) fn with_locked_io<C: Codec, R>( + &self, + f: impl FnOnce(&mut dyn Write, &mut dyn BufRead, &mut C::Buf) -> Result<R, ServerError>, + ) -> Result<R, ServerError> { + let state = &mut *self.state.lock().unwrap(); + let mut buf = C::Buf::default(); + + f(&mut state.stdin, &mut state.stdout, &mut buf).map_err(|e| { + if e.io.as_ref().map(|it| it.kind()) == Some(io::ErrorKind::BrokenPipe) { + match state.process.child.try_wait() { + Ok(None) | Err(_) => e, + Ok(Some(status)) => { + let mut msg = String::new(); + if !status.success() + && let Some(stderr) = state.process.child.stderr.as_mut() + { + _ = stderr.read_to_string(&mut msg); } + let server_error = ServerError { + message: format!( + "proc-macro server exited with {status}{}{msg}", + if msg.is_empty() { "" } else { ": " } + ), + io: None, + }; + self.exited.get_or_init(|| AssertUnwindSafe(server_error)).0.clone() } - } else { - e } - }) + } else { + e + } + }) + } + + pub(crate) fn run_bidirectional<C: Codec>( + &self, + initial: BidirectionalMessage, + callback: SubCallback<'_>, + ) -> Result<BidirectionalMessage, ServerError> { + self.with_locked_io::<C, _>(|writer, reader, buf| { + bidirectional_protocol::run_conversation::<C>(writer, reader, buf, initial, callback) + }) } } diff --git a/crates/proc-macro-api/src/transport.rs b/crates/proc-macro-api/src/transport.rs new file mode 100644 index 0000000000..b7a1d8f732 --- /dev/null +++ b/crates/proc-macro-api/src/transport.rs @@ -0,0 +1,3 @@ +//! Contains construct for transport of messages. +pub mod codec; +pub mod framing; diff --git a/crates/proc-macro-api/src/codec.rs b/crates/proc-macro-api/src/transport/codec.rs index baccaa6be4..c9afad260a 100644 --- a/crates/proc-macro-api/src/codec.rs +++ b/crates/proc-macro-api/src/transport/codec.rs @@ -4,7 +4,10 @@ use std::io; use serde::de::DeserializeOwned; -use crate::framing::Framing; +use crate::transport::framing::Framing; + +pub mod json; +pub mod postcard; pub trait Codec: Framing { fn encode<T: serde::Serialize>(msg: &T) -> io::Result<Self::Buf>; diff --git a/crates/proc-macro-api/src/legacy_protocol/json.rs b/crates/proc-macro-api/src/transport/codec/json.rs index 1359c05684..96db802e0b 100644 --- a/crates/proc-macro-api/src/legacy_protocol/json.rs +++ b/crates/proc-macro-api/src/transport/codec/json.rs @@ -3,14 +3,14 @@ use std::io::{self, BufRead, Write}; use serde::{Serialize, de::DeserializeOwned}; -use crate::{codec::Codec, framing::Framing}; +use crate::{Codec, transport::framing::Framing}; pub struct JsonProtocol; impl Framing for JsonProtocol { type Buf = String; - fn read<'a, R: BufRead>( + fn read<'a, R: BufRead + ?Sized>( inp: &mut R, buf: &'a mut String, ) -> io::Result<Option<&'a mut String>> { @@ -35,7 +35,7 @@ impl Framing for JsonProtocol { } } - fn write<W: Write>(out: &mut W, buf: &String) -> io::Result<()> { + fn write<W: Write + ?Sized>(out: &mut W, buf: &String) -> io::Result<()> { tracing::debug!("> {}", buf); out.write_all(buf.as_bytes())?; out.write_all(b"\n")?; diff --git a/crates/proc-macro-api/src/legacy_protocol/postcard.rs b/crates/proc-macro-api/src/transport/codec/postcard.rs index c28a9bfe3a..6f5319e75b 100644 --- a/crates/proc-macro-api/src/legacy_protocol/postcard.rs +++ b/crates/proc-macro-api/src/transport/codec/postcard.rs @@ -4,14 +4,14 @@ use std::io::{self, BufRead, Write}; use serde::{Serialize, de::DeserializeOwned}; -use crate::{codec::Codec, framing::Framing}; +use crate::{Codec, transport::framing::Framing}; pub struct PostcardProtocol; impl Framing for PostcardProtocol { type Buf = Vec<u8>; - fn read<'a, R: BufRead>( + fn read<'a, R: BufRead + ?Sized>( inp: &mut R, buf: &'a mut Vec<u8>, ) -> io::Result<Option<&'a mut Vec<u8>>> { @@ -23,7 +23,7 @@ impl Framing for PostcardProtocol { Ok(Some(buf)) } - fn write<W: Write>(out: &mut W, buf: &Vec<u8>) -> io::Result<()> { + fn write<W: Write + ?Sized>(out: &mut W, buf: &Vec<u8>) -> io::Result<()> { out.write_all(buf)?; out.flush() } diff --git a/crates/proc-macro-api/src/framing.rs b/crates/proc-macro-api/src/transport/framing.rs index a1e6fc05ca..2a11eb19c3 100644 --- a/crates/proc-macro-api/src/framing.rs +++ b/crates/proc-macro-api/src/transport/framing.rs @@ -5,10 +5,10 @@ use std::io::{self, BufRead, Write}; pub trait Framing { type Buf: Default; - fn read<'a, R: BufRead>( + fn read<'a, R: BufRead + ?Sized>( inp: &mut R, buf: &'a mut Self::Buf, ) -> io::Result<Option<&'a mut Self::Buf>>; - fn write<W: Write>(out: &mut W, buf: &Self::Buf) -> io::Result<()>; + fn write<W: Write + ?Sized>(out: &mut W, buf: &Self::Buf) -> io::Result<()>; } diff --git a/crates/proc-macro-srv-cli/Cargo.toml b/crates/proc-macro-srv-cli/Cargo.toml index 2c6e5a16ee..df3d21aefc 100644 --- a/crates/proc-macro-srv-cli/Cargo.toml +++ b/crates/proc-macro-srv-cli/Cargo.toml @@ -14,6 +14,7 @@ publish = false proc-macro-srv.workspace = true proc-macro-api.workspace = true postcard.workspace = true +crossbeam-channel.workspace = true clap = {version = "4.5.42", default-features = false, features = ["std"]} [features] diff --git a/crates/proc-macro-srv-cli/src/main.rs b/crates/proc-macro-srv-cli/src/main.rs index 813ac339a9..bdfdb50002 100644 --- a/crates/proc-macro-srv-cli/src/main.rs +++ b/crates/proc-macro-srv-cli/src/main.rs @@ -52,11 +52,16 @@ fn main() -> std::io::Result<()> { enum ProtocolFormat { JsonLegacy, PostcardLegacy, + BidirectionalPostcardPrototype, } impl ValueEnum for ProtocolFormat { fn value_variants<'a>() -> &'a [Self] { - &[ProtocolFormat::JsonLegacy, ProtocolFormat::PostcardLegacy] + &[ + ProtocolFormat::JsonLegacy, + ProtocolFormat::PostcardLegacy, + ProtocolFormat::BidirectionalPostcardPrototype, + ] } fn to_possible_value(&self) -> Option<clap::builder::PossibleValue> { @@ -65,12 +70,18 @@ impl ValueEnum for ProtocolFormat { ProtocolFormat::PostcardLegacy => { Some(clap::builder::PossibleValue::new("postcard-legacy")) } + ProtocolFormat::BidirectionalPostcardPrototype => { + Some(clap::builder::PossibleValue::new("bidirectional-postcard-prototype")) + } } } fn from_str(input: &str, _ignore_case: bool) -> Result<Self, String> { match input { "json-legacy" => Ok(ProtocolFormat::JsonLegacy), "postcard-legacy" => Ok(ProtocolFormat::PostcardLegacy), + "bidirectional-postcard-prototype" => { + Ok(ProtocolFormat::BidirectionalPostcardPrototype) + } _ => Err(format!("unknown protocol format: {input}")), } } diff --git a/crates/proc-macro-srv-cli/src/main_loop.rs b/crates/proc-macro-srv-cli/src/main_loop.rs index df54f38cbc..99e3d79ef2 100644 --- a/crates/proc-macro-srv-cli/src/main_loop.rs +++ b/crates/proc-macro-srv-cli/src/main_loop.rs @@ -3,22 +3,20 @@ use std::io; use proc_macro_api::{ Codec, - legacy_protocol::{ - json::JsonProtocol, - msg::{ - self, ExpandMacroData, ExpnGlobals, Message, SpanMode, SpanTransformer, - deserialize_span_data_index_map, serialize_span_data_index_map, - }, - postcard::PostcardProtocol, - }, + bidirectional_protocol::msg as bidirectional, + legacy_protocol::msg as legacy, + transport::codec::{json::JsonProtocol, postcard::PostcardProtocol}, version::CURRENT_API_VERSION, }; + +use legacy::Message; + use proc_macro_srv::{EnvSnapshot, SpanId}; use crate::ProtocolFormat; struct SpanTrans; -impl SpanTransformer for SpanTrans { +impl legacy::SpanTransformer for SpanTrans { type Table = (); type Span = SpanId; fn token_id_of( @@ -39,7 +37,250 @@ pub(crate) fn run(format: ProtocolFormat) -> io::Result<()> { match format { ProtocolFormat::JsonLegacy => run_::<JsonProtocol>(), ProtocolFormat::PostcardLegacy => run_::<PostcardProtocol>(), + ProtocolFormat::BidirectionalPostcardPrototype => run_new::<PostcardProtocol>(), + } +} + +fn run_new<C: Codec>() -> io::Result<()> { + fn macro_kind_to_api(kind: proc_macro_srv::ProcMacroKind) -> proc_macro_api::ProcMacroKind { + match kind { + proc_macro_srv::ProcMacroKind::CustomDerive => { + proc_macro_api::ProcMacroKind::CustomDerive + } + proc_macro_srv::ProcMacroKind::Bang => proc_macro_api::ProcMacroKind::Bang, + proc_macro_srv::ProcMacroKind::Attr => proc_macro_api::ProcMacroKind::Attr, + } + } + + let mut buf = C::Buf::default(); + let mut stdin = io::stdin().lock(); + let mut stdout = io::stdout().lock(); + + let env_snapshot = EnvSnapshot::default(); + let srv = proc_macro_srv::ProcMacroSrv::new(&env_snapshot); + + let mut span_mode = legacy::SpanMode::Id; + + 'outer: loop { + let req_opt = bidirectional::BidirectionalMessage::read::<_, C>(&mut stdin, &mut buf)?; + let Some(req) = req_opt else { + break 'outer; + }; + + match req { + bidirectional::BidirectionalMessage::Request(request) => match request { + bidirectional::Request::ListMacros { dylib_path } => { + let res = srv.list_macros(&dylib_path).map(|macros| { + macros + .into_iter() + .map(|(name, kind)| (name, macro_kind_to_api(kind))) + .collect() + }); + + send_response::<_, C>(&mut stdout, bidirectional::Response::ListMacros(res))?; + } + + bidirectional::Request::ApiVersionCheck {} => { + // bidirectional::Response::ApiVersionCheck(CURRENT_API_VERSION).write::<_, C>(stdout) + send_response::<_, C>( + &mut stdout, + bidirectional::Response::ApiVersionCheck(CURRENT_API_VERSION), + )?; + } + + bidirectional::Request::SetConfig(config) => { + span_mode = config.span_mode; + send_response::<_, C>(&mut stdout, bidirectional::Response::SetConfig(config))?; + } + bidirectional::Request::ExpandMacro(task) => { + handle_expand::<_, _, C>( + &srv, + &mut stdin, + &mut stdout, + &mut buf, + span_mode, + *task, + )?; + } + }, + _ => continue, + } } + + Ok(()) +} + +fn handle_expand<W: std::io::Write, R: std::io::BufRead, C: Codec>( + srv: &proc_macro_srv::ProcMacroSrv<'_>, + stdin: &mut R, + stdout: &mut W, + buf: &mut C::Buf, + span_mode: legacy::SpanMode, + task: bidirectional::ExpandMacro, +) -> io::Result<()> { + match span_mode { + legacy::SpanMode::Id => handle_expand_id::<_, C>(srv, stdout, task), + legacy::SpanMode::RustAnalyzer => { + handle_expand_ra::<_, _, C>(srv, stdin, stdout, buf, task) + } + } +} + +fn handle_expand_id<W: std::io::Write, C: Codec>( + srv: &proc_macro_srv::ProcMacroSrv<'_>, + stdout: &mut W, + task: bidirectional::ExpandMacro, +) -> io::Result<()> { + let bidirectional::ExpandMacro { lib, env, current_dir, data } = task; + let bidirectional::ExpandMacroData { + macro_body, + macro_name, + attributes, + has_global_spans: bidirectional::ExpnGlobals { def_site, call_site, mixed_site, .. }, + .. + } = data; + + let def_site = SpanId(def_site as u32); + let call_site = SpanId(call_site as u32); + let mixed_site = SpanId(mixed_site as u32); + + let macro_body = + macro_body.to_tokenstream_unresolved::<SpanTrans>(CURRENT_API_VERSION, |_, b| b); + let attributes = attributes + .map(|it| it.to_tokenstream_unresolved::<SpanTrans>(CURRENT_API_VERSION, |_, b| b)); + + let res = srv + .expand( + lib, + &env, + current_dir, + ¯o_name, + macro_body, + attributes, + def_site, + call_site, + mixed_site, + None, + ) + .map(|it| { + legacy::FlatTree::from_tokenstream_raw::<SpanTrans>(it, call_site, CURRENT_API_VERSION) + }) + .map_err(|e| legacy::PanicMessage(e.into_string().unwrap_or_default())); + + send_response::<_, C>(stdout, bidirectional::Response::ExpandMacro(res)) +} + +fn handle_expand_ra<W: io::Write, R: io::BufRead, C: Codec>( + srv: &proc_macro_srv::ProcMacroSrv<'_>, + stdin: &mut R, + stdout: &mut W, + buf: &mut C::Buf, + task: bidirectional::ExpandMacro, +) -> io::Result<()> { + let bidirectional::ExpandMacro { + lib, + env, + current_dir, + data: + bidirectional::ExpandMacroData { + macro_body, + macro_name, + attributes, + has_global_spans: bidirectional::ExpnGlobals { def_site, call_site, mixed_site, .. }, + span_data_table, + }, + } = task; + + let mut span_data_table = legacy::deserialize_span_data_index_map(&span_data_table); + + let def_site = span_data_table[def_site]; + let call_site = span_data_table[call_site]; + let mixed_site = span_data_table[mixed_site]; + + let macro_body = + macro_body.to_tokenstream_resolved(CURRENT_API_VERSION, &span_data_table, |a, b| { + srv.join_spans(a, b).unwrap_or(b) + }); + let attributes = attributes.map(|it| { + it.to_tokenstream_resolved(CURRENT_API_VERSION, &span_data_table, |a, b| { + srv.join_spans(a, b).unwrap_or(b) + }) + }); + + let (subreq_tx, subreq_rx) = crossbeam_channel::unbounded(); + let (subresp_tx, subresp_rx) = crossbeam_channel::unbounded(); + let (result_tx, result_rx) = crossbeam_channel::bounded(1); + + std::thread::scope(|scope| { + scope.spawn(|| { + let callback = Box::new(move |req: proc_macro_srv::SubRequest| { + subreq_tx.send(req).unwrap(); + subresp_rx.recv().unwrap() + }); + + let res = srv + .expand( + lib, + &env, + current_dir, + ¯o_name, + macro_body, + attributes, + def_site, + call_site, + mixed_site, + Some(callback), + ) + .map(|it| { + ( + legacy::FlatTree::from_tokenstream( + it, + CURRENT_API_VERSION, + call_site, + &mut span_data_table, + ), + legacy::serialize_span_data_index_map(&span_data_table), + ) + }) + .map(|(tree, span_data_table)| bidirectional::ExpandMacroExtended { + tree, + span_data_table, + }) + .map_err(|e| legacy::PanicMessage(e.into_string().unwrap_or_default())); + + let _ = result_tx.send(res); + }); + + loop { + if let Ok(res) = result_rx.try_recv() { + send_response::<_, C>(stdout, bidirectional::Response::ExpandMacroExtended(res)) + .unwrap(); + break; + } + + let subreq = match subreq_rx.recv() { + Ok(r) => r, + Err(_) => break, + }; + + let api_req = from_srv_req(subreq); + bidirectional::BidirectionalMessage::SubRequest(api_req).write::<_, C>(stdout).unwrap(); + + let resp = bidirectional::BidirectionalMessage::read::<_, C>(stdin, buf) + .unwrap() + .expect("client closed connection"); + + match resp { + bidirectional::BidirectionalMessage::SubResponse(api_resp) => { + let srv_resp = from_client_res(api_resp); + subresp_tx.send(srv_resp).unwrap(); + } + other => panic!("expected SubResponse, got {other:?}"), + } + } + }); + + Ok(()) } fn run_<C: Codec>() -> io::Result<()> { @@ -54,38 +295,38 @@ fn run_<C: Codec>() -> io::Result<()> { } let mut buf = C::Buf::default(); - let mut read_request = || msg::Request::read::<_, C>(&mut io::stdin().lock(), &mut buf); - let write_response = |msg: msg::Response| msg.write::<_, C>(&mut io::stdout().lock()); + let mut read_request = || legacy::Request::read::<_, C>(&mut io::stdin().lock(), &mut buf); + let write_response = |msg: legacy::Response| msg.write::<_, C>(&mut io::stdout().lock()); let env = EnvSnapshot::default(); let srv = proc_macro_srv::ProcMacroSrv::new(&env); - let mut span_mode = SpanMode::Id; + let mut span_mode = legacy::SpanMode::Id; while let Some(req) = read_request()? { let res = match req { - msg::Request::ListMacros { dylib_path } => { - msg::Response::ListMacros(srv.list_macros(&dylib_path).map(|macros| { + legacy::Request::ListMacros { dylib_path } => { + legacy::Response::ListMacros(srv.list_macros(&dylib_path).map(|macros| { macros.into_iter().map(|(name, kind)| (name, macro_kind_to_api(kind))).collect() })) } - msg::Request::ExpandMacro(task) => { - let msg::ExpandMacro { + legacy::Request::ExpandMacro(task) => { + let legacy::ExpandMacro { lib, env, current_dir, data: - ExpandMacroData { + legacy::ExpandMacroData { macro_body, macro_name, attributes, has_global_spans: - ExpnGlobals { serialize: _, def_site, call_site, mixed_site }, + legacy::ExpnGlobals { serialize: _, def_site, call_site, mixed_site }, span_data_table, }, } = *task; match span_mode { - SpanMode::Id => msg::Response::ExpandMacro({ + legacy::SpanMode::Id => legacy::Response::ExpandMacro({ let def_site = SpanId(def_site as u32); let call_site = SpanId(call_site as u32); let mixed_site = SpanId(mixed_site as u32); @@ -106,19 +347,21 @@ fn run_<C: Codec>() -> io::Result<()> { def_site, call_site, mixed_site, + None, ) .map(|it| { - msg::FlatTree::from_tokenstream_raw::<SpanTrans>( + legacy::FlatTree::from_tokenstream_raw::<SpanTrans>( it, call_site, CURRENT_API_VERSION, ) }) .map_err(|e| e.into_string().unwrap_or_default()) - .map_err(msg::PanicMessage) + .map_err(legacy::PanicMessage) }), - SpanMode::RustAnalyzer => msg::Response::ExpandMacroExtended({ - let mut span_data_table = deserialize_span_data_index_map(&span_data_table); + legacy::SpanMode::RustAnalyzer => legacy::Response::ExpandMacroExtended({ + let mut span_data_table = + legacy::deserialize_span_data_index_map(&span_data_table); let def_site = span_data_table[def_site]; let call_site = span_data_table[call_site]; @@ -146,31 +389,34 @@ fn run_<C: Codec>() -> io::Result<()> { def_site, call_site, mixed_site, + None, ) .map(|it| { ( - msg::FlatTree::from_tokenstream( + legacy::FlatTree::from_tokenstream( it, CURRENT_API_VERSION, call_site, &mut span_data_table, ), - serialize_span_data_index_map(&span_data_table), + legacy::serialize_span_data_index_map(&span_data_table), ) }) - .map(|(tree, span_data_table)| msg::ExpandMacroExtended { + .map(|(tree, span_data_table)| legacy::ExpandMacroExtended { tree, span_data_table, }) .map_err(|e| e.into_string().unwrap_or_default()) - .map_err(msg::PanicMessage) + .map_err(legacy::PanicMessage) }), } } - msg::Request::ApiVersionCheck {} => msg::Response::ApiVersionCheck(CURRENT_API_VERSION), - msg::Request::SetConfig(config) => { + legacy::Request::ApiVersionCheck {} => { + legacy::Response::ApiVersionCheck(CURRENT_API_VERSION) + } + legacy::Request::SetConfig(config) => { span_mode = config.span_mode; - msg::Response::SetConfig(config) + legacy::Response::SetConfig(config) } }; write_response(res)? @@ -178,3 +424,27 @@ fn run_<C: Codec>() -> io::Result<()> { Ok(()) } + +fn from_srv_req(value: proc_macro_srv::SubRequest) -> bidirectional::SubRequest { + match value { + proc_macro_srv::SubRequest::SourceText { file_id, start, end } => { + bidirectional::SubRequest::SourceText { file_id: file_id.file_id().index(), start, end } + } + } +} + +fn from_client_res(value: bidirectional::SubResponse) -> proc_macro_srv::SubResponse { + match value { + bidirectional::SubResponse::SourceTextResult { text } => { + proc_macro_srv::SubResponse::SourceTextResult { text } + } + } +} + +fn send_response<W: std::io::Write, C: Codec>( + stdout: &mut W, + resp: bidirectional::Response, +) -> io::Result<()> { + let resp = bidirectional::BidirectionalMessage::Response(resp); + resp.write::<W, C>(stdout) +} diff --git a/crates/proc-macro-srv/src/dylib.rs b/crates/proc-macro-srv/src/dylib.rs index 03433197b7..082a1d77b5 100644 --- a/crates/proc-macro-srv/src/dylib.rs +++ b/crates/proc-macro-srv/src/dylib.rs @@ -12,7 +12,7 @@ use object::Object; use paths::{Utf8Path, Utf8PathBuf}; use crate::{ - PanicMessage, ProcMacroKind, ProcMacroSrvSpan, dylib::proc_macros::ProcMacros, + PanicMessage, ProcMacroKind, ProcMacroSrvSpan, SubCallback, dylib::proc_macros::ProcMacros, token_stream::TokenStream, }; @@ -45,13 +45,14 @@ impl Expander { def_site: S, call_site: S, mixed_site: S, + callback: Option<SubCallback>, ) -> Result<TokenStream<S>, PanicMessage> where <S::Server as bridge::server::Types>::TokenStream: Default, { self.inner .proc_macros - .expand(macro_name, macro_body, attribute, def_site, call_site, mixed_site) + .expand(macro_name, macro_body, attribute, def_site, call_site, mixed_site, callback) } pub(crate) fn list_macros(&self) -> impl Iterator<Item = (&str, ProcMacroKind)> { diff --git a/crates/proc-macro-srv/src/dylib/proc_macros.rs b/crates/proc-macro-srv/src/dylib/proc_macros.rs index c879c7609d..6f6bd086de 100644 --- a/crates/proc-macro-srv/src/dylib/proc_macros.rs +++ b/crates/proc-macro-srv/src/dylib/proc_macros.rs @@ -1,9 +1,7 @@ //! Proc macro ABI - +use crate::{ProcMacroKind, ProcMacroSrvSpan, SubCallback, token_stream::TokenStream}; use proc_macro::bridge; -use crate::{ProcMacroKind, ProcMacroSrvSpan, token_stream::TokenStream}; - #[repr(transparent)] pub(crate) struct ProcMacros([bridge::client::ProcMacro]); @@ -22,6 +20,7 @@ impl ProcMacros { def_site: S, call_site: S, mixed_site: S, + callback: Option<SubCallback>, ) -> Result<TokenStream<S>, crate::PanicMessage> { let parsed_attributes = attribute.unwrap_or_default(); @@ -32,7 +31,7 @@ impl ProcMacros { { let res = client.run( &bridge::server::SameThread, - S::make_server(call_site, def_site, mixed_site), + S::make_server(call_site, def_site, mixed_site, callback), macro_body, cfg!(debug_assertions), ); @@ -41,7 +40,7 @@ impl ProcMacros { bridge::client::ProcMacro::Bang { name, client } if *name == macro_name => { let res = client.run( &bridge::server::SameThread, - S::make_server(call_site, def_site, mixed_site), + S::make_server(call_site, def_site, mixed_site, callback), macro_body, cfg!(debug_assertions), ); @@ -50,7 +49,7 @@ impl ProcMacros { bridge::client::ProcMacro::Attr { name, client } if *name == macro_name => { let res = client.run( &bridge::server::SameThread, - S::make_server(call_site, def_site, mixed_site), + S::make_server(call_site, def_site, mixed_site, callback), parsed_attributes, macro_body, cfg!(debug_assertions), diff --git a/crates/proc-macro-srv/src/lib.rs b/crates/proc-macro-srv/src/lib.rs index 93319df824..705ac930ed 100644 --- a/crates/proc-macro-srv/src/lib.rs +++ b/crates/proc-macro-srv/src/lib.rs @@ -47,7 +47,7 @@ use std::{ }; use paths::{Utf8Path, Utf8PathBuf}; -use span::Span; +use span::{EditionedFileId, Span}; use temp_dir::TempDir; pub use crate::server_impl::token_id::SpanId; @@ -91,6 +91,16 @@ impl<'env> ProcMacroSrv<'env> { } } +pub type SubCallback = Box<dyn Fn(SubRequest) -> SubResponse + Send + Sync + 'static>; + +pub enum SubRequest { + SourceText { file_id: EditionedFileId, start: u32, end: u32 }, +} + +pub enum SubResponse { + SourceTextResult { text: Option<String> }, +} + const EXPANDER_STACK_SIZE: usize = 8 * 1024 * 1024; impl ProcMacroSrv<'_> { @@ -105,6 +115,7 @@ impl ProcMacroSrv<'_> { def_site: S, call_site: S, mixed_site: S, + callback: Option<SubCallback>, ) -> Result<token_stream::TokenStream<S>, PanicMessage> { let snapped_env = self.env; let expander = self.expander(lib.as_ref()).map_err(|err| PanicMessage { @@ -120,8 +131,10 @@ impl ProcMacroSrv<'_> { .stack_size(EXPANDER_STACK_SIZE) .name(macro_name.to_owned()) .spawn_scoped(s, move || { - expander - .expand(macro_name, macro_body, attribute, def_site, call_site, mixed_site) + expander.expand( + macro_name, macro_body, attribute, def_site, call_site, mixed_site, + callback, + ) }); match thread.unwrap().join() { Ok(res) => res, @@ -170,29 +183,47 @@ impl ProcMacroSrv<'_> { pub trait ProcMacroSrvSpan: Copy + Send + Sync { type Server: proc_macro::bridge::server::Server<TokenStream = crate::token_stream::TokenStream<Self>>; - fn make_server(call_site: Self, def_site: Self, mixed_site: Self) -> Self::Server; + fn make_server( + call_site: Self, + def_site: Self, + mixed_site: Self, + callback: Option<SubCallback>, + ) -> Self::Server; } impl ProcMacroSrvSpan for SpanId { type Server = server_impl::token_id::SpanIdServer; - fn make_server(call_site: Self, def_site: Self, mixed_site: Self) -> Self::Server { + fn make_server( + call_site: Self, + def_site: Self, + mixed_site: Self, + callback: Option<SubCallback>, + ) -> Self::Server { Self::Server { call_site, def_site, mixed_site, + callback, tracked_env_vars: Default::default(), tracked_paths: Default::default(), } } } + impl ProcMacroSrvSpan for Span { type Server = server_impl::rust_analyzer_span::RaSpanServer; - fn make_server(call_site: Self, def_site: Self, mixed_site: Self) -> Self::Server { + fn make_server( + call_site: Self, + def_site: Self, + mixed_site: Self, + callback: Option<SubCallback>, + ) -> Self::Server { Self::Server { call_site, def_site, mixed_site, + callback, tracked_env_vars: Default::default(), tracked_paths: Default::default(), } diff --git a/crates/proc-macro-srv/src/server_impl/rust_analyzer_span.rs b/crates/proc-macro-srv/src/server_impl/rust_analyzer_span.rs index 7c685c2da7..0bce67fcd9 100644 --- a/crates/proc-macro-srv/src/server_impl/rust_analyzer_span.rs +++ b/crates/proc-macro-srv/src/server_impl/rust_analyzer_span.rs @@ -14,6 +14,7 @@ use proc_macro::bridge::server; use span::{FIXUP_ERASED_FILE_AST_ID_MARKER, Span, TextRange, TextSize}; use crate::{ + SubCallback, SubRequest, SubResponse, bridge::{Diagnostic, ExpnGlobals, Literal, TokenTree}, server_impl::literal_from_str, }; @@ -28,6 +29,7 @@ pub struct RaSpanServer { pub call_site: Span, pub def_site: Span, pub mixed_site: Span, + pub callback: Option<SubCallback>, } impl server::Types for RaSpanServer { @@ -149,9 +151,19 @@ impl server::Span for RaSpanServer { /// /// See PR: /// https://github.com/rust-lang/rust/pull/55780 - fn source_text(&mut self, _span: Self::Span) -> Option<String> { - // FIXME requires db, needs special handling wrt fixup spans - None + fn source_text(&mut self, span: Self::Span) -> Option<String> { + let file_id = span.anchor.file_id; + let start: u32 = span.range.start().into(); + let end: u32 = span.range.end().into(); + + let req = SubRequest::SourceText { file_id, start, end }; + + let cb = self.callback.as_mut()?; + let response = cb(req); + + match response { + SubResponse::SourceTextResult { text } => text, + } } fn parent(&mut self, _span: Self::Span) -> Option<Self::Span> { diff --git a/crates/proc-macro-srv/src/server_impl/token_id.rs b/crates/proc-macro-srv/src/server_impl/token_id.rs index 5ac263b9d5..9db7597d84 100644 --- a/crates/proc-macro-srv/src/server_impl/token_id.rs +++ b/crates/proc-macro-srv/src/server_impl/token_id.rs @@ -9,6 +9,7 @@ use intern::Symbol; use proc_macro::bridge::server; use crate::{ + SubCallback, bridge::{Diagnostic, ExpnGlobals, Literal, TokenTree}, server_impl::literal_from_str, }; @@ -34,6 +35,7 @@ pub struct SpanIdServer { pub call_site: Span, pub def_site: Span, pub mixed_site: Span, + pub callback: Option<SubCallback>, } impl server::Types for SpanIdServer { diff --git a/crates/proc-macro-srv/src/tests/utils.rs b/crates/proc-macro-srv/src/tests/utils.rs index 1b12308ad6..61fcd810b1 100644 --- a/crates/proc-macro-srv/src/tests/utils.rs +++ b/crates/proc-macro-srv/src/tests/utils.rs @@ -59,8 +59,9 @@ fn assert_expand_impl( let input_ts_string = format!("{input_ts:?}"); let attr_ts_string = attr_ts.as_ref().map(|it| format!("{it:?}")); - let res = - expander.expand(macro_name, input_ts, attr_ts, def_site, call_site, mixed_site).unwrap(); + let res = expander + .expand(macro_name, input_ts, attr_ts, def_site, call_site, mixed_site, None) + .unwrap(); expect.assert_eq(&format!( "{input_ts_string}{}{}{}", if attr_ts_string.is_some() { "\n\n" } else { "" }, @@ -91,7 +92,8 @@ fn assert_expand_impl( let fixture_string = format!("{fixture:?}"); let attr_string = attr.as_ref().map(|it| format!("{it:?}")); - let res = expander.expand(macro_name, fixture, attr, def_site, call_site, mixed_site).unwrap(); + let res = + expander.expand(macro_name, fixture, attr, def_site, call_site, mixed_site, None).unwrap(); expect_spanned.assert_eq(&format!( "{fixture_string}{}{}{}", if attr_string.is_some() { "\n\n" } else { "" }, diff --git a/crates/test-fixture/src/lib.rs b/crates/test-fixture/src/lib.rs index 01e4215cfb..67f69d0fa9 100644 --- a/crates/test-fixture/src/lib.rs +++ b/crates/test-fixture/src/lib.rs @@ -738,6 +738,7 @@ struct IdentityProcMacroExpander; impl ProcMacroExpander for IdentityProcMacroExpander { fn expand( &self, + _: &dyn SourceDatabase, subtree: &TopSubtree, _: Option<&TopSubtree>, _: &Env, @@ -760,6 +761,7 @@ struct Issue18089ProcMacroExpander; impl ProcMacroExpander for Issue18089ProcMacroExpander { fn expand( &self, + _: &dyn SourceDatabase, subtree: &TopSubtree, _: Option<&TopSubtree>, _: &Env, @@ -795,6 +797,7 @@ struct AttributeInputReplaceProcMacroExpander; impl ProcMacroExpander for AttributeInputReplaceProcMacroExpander { fn expand( &self, + _: &dyn SourceDatabase, _: &TopSubtree, attrs: Option<&TopSubtree>, _: &Env, @@ -818,6 +821,7 @@ struct Issue18840ProcMacroExpander; impl ProcMacroExpander for Issue18840ProcMacroExpander { fn expand( &self, + _: &dyn SourceDatabase, fn_: &TopSubtree, _: Option<&TopSubtree>, _: &Env, @@ -853,6 +857,7 @@ struct MirrorProcMacroExpander; impl ProcMacroExpander for MirrorProcMacroExpander { fn expand( &self, + _: &dyn SourceDatabase, input: &TopSubtree, _: Option<&TopSubtree>, _: &Env, @@ -891,6 +896,7 @@ struct ShortenProcMacroExpander; impl ProcMacroExpander for ShortenProcMacroExpander { fn expand( &self, + _: &dyn SourceDatabase, input: &TopSubtree, _: Option<&TopSubtree>, _: &Env, @@ -933,6 +939,7 @@ struct Issue17479ProcMacroExpander; impl ProcMacroExpander for Issue17479ProcMacroExpander { fn expand( &self, + _: &dyn SourceDatabase, subtree: &TopSubtree, _: Option<&TopSubtree>, _: &Env, @@ -962,6 +969,7 @@ struct Issue18898ProcMacroExpander; impl ProcMacroExpander for Issue18898ProcMacroExpander { fn expand( &self, + _: &dyn SourceDatabase, subtree: &TopSubtree, _: Option<&TopSubtree>, _: &Env, @@ -1017,6 +1025,7 @@ struct DisallowCfgProcMacroExpander; impl ProcMacroExpander for DisallowCfgProcMacroExpander { fn expand( &self, + _: &dyn SourceDatabase, subtree: &TopSubtree, _: Option<&TopSubtree>, _: &Env, @@ -1048,6 +1057,7 @@ struct GenerateSuffixedTypeProcMacroExpander; impl ProcMacroExpander for GenerateSuffixedTypeProcMacroExpander { fn expand( &self, + _: &dyn SourceDatabase, subtree: &TopSubtree, _attrs: Option<&TopSubtree>, _env: &Env, |