Unnamed repository; edit this file 'description' to name the repository.
Merge pull request #20280 from Kobzol/josh-sync
Switch to using josh-sync
| -rw-r--r-- | Cargo.lock | 11 | ||||
| -rw-r--r-- | docs/book/src/contributing/README.md | 54 | ||||
| -rw-r--r-- | xtask/Cargo.toml | 1 | ||||
| -rw-r--r-- | xtask/src/flags.rs | 28 | ||||
| -rw-r--r-- | xtask/src/main.rs | 2 | ||||
| -rw-r--r-- | xtask/src/release.rs | 175 |
6 files changed, 42 insertions, 229 deletions
diff --git a/Cargo.lock b/Cargo.lock index c471234bbe..a699463ca4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -396,15 +396,6 @@ dependencies = [ ] [[package]] -name = "directories" -version = "6.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16f5094c54661b38d03bd7e50df373292118db60b585c08a411c6d840017fe7d" -dependencies = [ - "dirs-sys", -] - -[[package]] name = "dirs" version = "6.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -3105,7 +3096,6 @@ name = "xtask" version = "0.1.0" dependencies = [ "anyhow", - "directories", "edition", "either", "flate2", @@ -3113,7 +3103,6 @@ dependencies = [ "proc-macro2", "quote", "stdx", - "time", "ungrammar", "write-json", "xflags", diff --git a/docs/book/src/contributing/README.md b/docs/book/src/contributing/README.md index beb94cdfc4..57c7a9c599 100644 --- a/docs/book/src/contributing/README.md +++ b/docs/book/src/contributing/README.md @@ -252,18 +252,8 @@ Release steps: 4. Commit & push the changelog. 5. Run `cargo xtask publish-release-notes <CHANGELOG>` -- this will convert the changelog entry in AsciiDoc to Markdown and update the body of GitHub Releases entry. 6. Tweet. -7. Make a new branch and run `cargo xtask rustc-pull`, open a PR, and merge it. - This will pull any changes from `rust-lang/rust` into `rust-analyzer`. -8. Switch to `master`, pull, then run `cargo xtask rustc-push --rust-path ../rust-rust-analyzer --rust-fork matklad/rust`. - Replace `matklad/rust` with your own fork of `rust-lang/rust`. - You can use the token to authenticate when you get prompted for a password, since `josh` will push over HTTPS, not SSH. - This will push the `rust-analyzer` changes to your fork. - You can then open a PR against `rust-lang/rust`. - -Note: besides the `rust-rust-analyzer` clone, the Josh cache (stored under `~/.cache/rust-analyzer-josh`) will contain a bare clone of `rust-lang/rust`. -This currently takes about 3.5 GB. - -This [HackMD](https://hackmd.io/7pOuxnkdQDaL1Y1FQr65xg) has details about how `josh` syncs work. +7. Perform a subtree [pull](#performing-a-pull). +8. Perform a subtree [push](#performing-a-push). If the GitHub Actions release fails because of a transient problem like a timeout, you can re-run the job from the Actions console. If it fails because of something that needs to be fixed, remove the release tag (if needed), fix the problem, then start over. @@ -288,3 +278,43 @@ There are two sets of people with extra permissions: If you don't feel like reviewing for whatever reason, someone else will pick the review up (but please speak up if you don't feel like it)! * The [rust-lang](https://github.com/rust-lang) team [t-rust-analyzer-contributors]([https://github.com/orgs/rust-analyzer/teams/triage](https://github.com/rust-lang/team/blob/master/teams/rust-analyzer-contributors.toml)). This team has general triaging permissions allowing to label, close and re-open issues. + +## Synchronizing subtree changes +`rust-analyzer` is a [josh](https://josh-project.github.io/josh/intro.html) subtree of the [rust-lang/rust](https://github.com/rust-lang/rust) +repository. We use the [rustc-josh-sync](https://github.com/rust-lang/josh-sync) tool to perform synchronization between these two +repositories. You can find documentation of the tool [here](https://github.com/rust-lang/josh-sync). + +You can install the synchronization tool using the following commands: +``` +cargo install --locked --git https://github.com/rust-lang/josh-sync +``` + +Both pulls (synchronizing changes from rust-lang/rust into rust-analyzer) and pushes (synchronizing +changes from rust-analyzer into rust-lang/rust) are performed from this repository. +changes from rust-analyzer to rust-lang/rust) are performed from this repository. + +Usually we first perform a pull, wait for it to be merged, and then perform a push. + +### Performing a pull +1) Checkout a new branch that will be used to create a PR against rust-analyzer +2) Run the pull command + ``` + rustc-josh-sync pull + ``` +3) Push the branch to your fork of `rust-analyzer` and create a PR + - If you have the `gh` CLI installed, `rustc-josh-sync` can create the PR for you. + +### Performing a push + +Wait for the previous pull to be merged. + +1) Switch to `master` and pull +2) Run the push command to create a branch named `<branch-name>` in a `rustc` fork under the `<gh-username>` account + ``` + rustc-josh-sync push <branch-name> <gh-username> + ``` + - The push will ask you to download a checkout of the `rust-lang/rust` repository. + - If you get prompted for a password, see [this](https://github.com/rust-lang/josh-sync?tab=readme-ov-file#git-peculiarities). +3) Create a PR from `<branch-name>` into `rust-lang/rust` + +> Besides the `rust` checkout, the Josh cache (stored under `~/.cache/rustc-josh`) will contain a bare clone of `rust-lang/rust`. This currently takes several GBs. diff --git a/xtask/Cargo.toml b/xtask/Cargo.toml index 8cd5811c0a..9d8a1956d0 100644 --- a/xtask/Cargo.toml +++ b/xtask/Cargo.toml @@ -8,7 +8,6 @@ rust-version.workspace = true [dependencies] anyhow.workspace = true -directories = "6.0" flate2 = "1.1.2" write-json = "0.1.4" xshell.workspace = true diff --git a/xtask/src/flags.rs b/xtask/src/flags.rs index 2fd471b35c..72f6215d4c 100644 --- a/xtask/src/flags.rs +++ b/xtask/src/flags.rs @@ -59,20 +59,6 @@ xflags::xflags! { optional --dry-run } - cmd rustc-pull { - /// rustc commit to pull. - optional --commit refspec: String - } - - cmd rustc-push { - /// rust local path, e.g. `../rust-rust-analyzer`. - required --rust-path rust_path: String - /// rust fork name, e.g. `matklad/rust`. - required --rust-fork rust_fork: String - /// branch name. - optional --branch branch: String - } - cmd dist { /// Use mimalloc allocator for server optional --mimalloc @@ -121,8 +107,6 @@ pub enum XtaskCmd { Install(Install), FuzzTests(FuzzTests), Release(Release), - RustcPull(RustcPull), - RustcPush(RustcPush), Dist(Dist), PublishReleaseNotes(PublishReleaseNotes), Metrics(Metrics), @@ -152,18 +136,6 @@ pub struct Release { } #[derive(Debug)] -pub struct RustcPull { - pub commit: Option<String>, -} - -#[derive(Debug)] -pub struct RustcPush { - pub rust_path: String, - pub rust_fork: String, - pub branch: Option<String>, -} - -#[derive(Debug)] pub struct Dist { pub mimalloc: bool, pub jemalloc: bool, diff --git a/xtask/src/main.rs b/xtask/src/main.rs index aaa8d0e1d4..c5ad49cdce 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -42,8 +42,6 @@ fn main() -> anyhow::Result<()> { flags::XtaskCmd::Install(cmd) => cmd.run(sh), flags::XtaskCmd::FuzzTests(_) => run_fuzzer(sh), flags::XtaskCmd::Release(cmd) => cmd.run(sh), - flags::XtaskCmd::RustcPull(cmd) => cmd.run(sh), - flags::XtaskCmd::RustcPush(cmd) => cmd.run(sh), flags::XtaskCmd::Dist(cmd) => cmd.run(sh), flags::XtaskCmd::PublishReleaseNotes(cmd) => cmd.run(sh), flags::XtaskCmd::Metrics(cmd) => cmd.run(sh), diff --git a/xtask/src/release.rs b/xtask/src/release.rs index e41f4ceb43..d06a25c892 100644 --- a/xtask/src/release.rs +++ b/xtask/src/release.rs @@ -1,12 +1,5 @@ mod changelog; -use std::process::{Command, Stdio}; -use std::thread; -use std::time::Duration; - -use anyhow::{Context as _, bail}; -use directories::ProjectDirs; -use stdx::JodChild; use xshell::{Shell, cmd}; use crate::{date_iso, flags, is_release_tag, project_root}; @@ -59,171 +52,3 @@ impl flags::Release { Ok(()) } } - -// git sync implementation adapted from https://github.com/rust-lang/miri/blob/62039ac/miri-script/src/commands.rs -impl flags::RustcPull { - pub(crate) fn run(self, sh: &Shell) -> anyhow::Result<()> { - sh.change_dir(project_root()); - let commit = self.commit.map(Result::Ok).unwrap_or_else(|| { - let rust_repo_head = - cmd!(sh, "git ls-remote https://github.com/rust-lang/rust/ HEAD").read()?; - rust_repo_head - .split_whitespace() - .next() - .map(|front| front.trim().to_owned()) - .ok_or_else(|| anyhow::format_err!("Could not obtain Rust repo HEAD from remote.")) - })?; - // Make sure the repo is clean. - if !cmd!(sh, "git status --untracked-files=no --porcelain").read()?.is_empty() { - bail!("working directory must be clean before running `cargo xtask pull`"); - } - // This should not add any new root commits. So count those before and after merging. - let num_roots = || -> anyhow::Result<u32> { - Ok(cmd!(sh, "git rev-list HEAD --max-parents=0 --count") - .read() - .context("failed to determine the number of root commits")? - .parse::<u32>()?) - }; - let num_roots_before = num_roots()?; - // Make sure josh is running. - let josh = start_josh()?; - - // Update rust-version file. As a separate commit, since making it part of - // the merge has confused the heck out of josh in the past. - // We pass `--no-verify` to avoid running any git hooks that might exist, - // in case they dirty the repository. - sh.write_file("rust-version", format!("{commit}\n"))?; - const PREPARING_COMMIT_MESSAGE: &str = "Preparing for merge from rust-lang/rust"; - cmd!(sh, "git commit rust-version --no-verify -m {PREPARING_COMMIT_MESSAGE}") - .run() - .context("FAILED to commit rust-version file, something went wrong")?; - - // Fetch given rustc commit. - cmd!(sh, "git fetch http://localhost:{JOSH_PORT}/rust-lang/rust.git@{commit}{JOSH_FILTER}.git") - .run() - .inspect_err(|_| { - // Try to un-do the previous `git commit`, to leave the repo in the state we found it it. - cmd!(sh, "git reset --hard HEAD^") - .run() - .expect("FAILED to clean up again after failed `git fetch`, sorry for that"); - }) - .context("FAILED to fetch new commits, something went wrong (committing the rust-version file has been undone)")?; - - // Merge the fetched commit. - const MERGE_COMMIT_MESSAGE: &str = "Merge from rust-lang/rust"; - cmd!(sh, "git merge FETCH_HEAD --no-verify --no-ff -m {MERGE_COMMIT_MESSAGE}") - .run() - .context("FAILED to merge new commits, something went wrong")?; - - // Check that the number of roots did not increase. - if num_roots()? != num_roots_before { - bail!("Josh created a new root commit. This is probably not the history you want."); - } - - drop(josh); - Ok(()) - } -} - -impl flags::RustcPush { - pub(crate) fn run(self, sh: &Shell) -> anyhow::Result<()> { - let branch = self.branch.as_deref().unwrap_or("sync-from-ra"); - let rust_path = self.rust_path; - let rust_fork = self.rust_fork; - - sh.change_dir(project_root()); - let base = sh.read_file("rust-version")?.trim().to_owned(); - // Make sure the repo is clean. - if !cmd!(sh, "git status --untracked-files=no --porcelain").read()?.is_empty() { - bail!("working directory must be clean before running `cargo xtask push`"); - } - // Make sure josh is running. - let josh = start_josh()?; - - // Find a repo we can do our preparation in. - sh.change_dir(rust_path); - - // Prepare the branch. Pushing works much better if we use as base exactly - // the commit that we pulled from last time, so we use the `rust-version` - // file to find out which commit that would be. - println!("Preparing {rust_fork} (base: {base})..."); - if cmd!(sh, "git fetch https://github.com/{rust_fork} {branch}") - .ignore_stderr() - .read() - .is_ok() - { - bail!( - "The branch `{branch}` seems to already exist in `https://github.com/{rust_fork}`. Please delete it and try again." - ); - } - cmd!(sh, "git fetch https://github.com/rust-lang/rust {base}").run()?; - cmd!(sh, "git push https://github.com/{rust_fork} {base}:refs/heads/{branch}") - .ignore_stdout() - .ignore_stderr() // silence the "create GitHub PR" message - .run()?; - println!(); - - // Do the actual push. - sh.change_dir(project_root()); - println!("Pushing rust-analyzer changes..."); - cmd!( - sh, - "git push http://localhost:{JOSH_PORT}/{rust_fork}.git{JOSH_FILTER}.git HEAD:{branch}" - ) - .run()?; - println!(); - - // Do a round-trip check to make sure the push worked as expected. - cmd!( - sh, - "git fetch http://localhost:{JOSH_PORT}/{rust_fork}.git{JOSH_FILTER}.git {branch}" - ) - .ignore_stderr() - .read()?; - let head = cmd!(sh, "git rev-parse HEAD").read()?; - let fetch_head = cmd!(sh, "git rev-parse FETCH_HEAD").read()?; - if head != fetch_head { - bail!( - "Josh created a non-roundtrip push! Do NOT merge this into rustc!\n\ - Expected {head}, got {fetch_head}." - ); - } - println!( - "Confirmed that the push round-trips back to rust-analyzer properly. Please create a rustc PR:" - ); - // https://github.com/github-linguist/linguist/compare/master...octocat:linguist:master - let fork_path = rust_fork.replace('/', ":"); - println!( - " https://github.com/rust-lang/rust/compare/{fork_path}:{branch}?quick_pull=1&title=Subtree+update+of+rust-analyzer&body=r?+@ghost" - ); - - drop(josh); - Ok(()) - } -} - -/// Used for rustc syncs. -const JOSH_FILTER: &str = ":rev(55d9a533b309119c8acd13061581b43ae8840823:prefix=src/tools/rust-analyzer):/src/tools/rust-analyzer"; -const JOSH_PORT: &str = "42042"; - -fn start_josh() -> anyhow::Result<impl Drop> { - // Determine cache directory. - let local_dir = { - let user_dirs = ProjectDirs::from("org", "rust-lang", "rust-analyzer-josh").unwrap(); - user_dirs.cache_dir().to_owned() - }; - - // Start josh, silencing its output. - let mut cmd = Command::new("josh-proxy"); - cmd.arg("--local").arg(local_dir); - cmd.arg("--remote").arg("https://github.com"); - cmd.arg("--port").arg(JOSH_PORT); - cmd.arg("--no-background"); - cmd.stdout(Stdio::null()); - cmd.stderr(Stdio::null()); - let josh = cmd.spawn().context("failed to start josh-proxy, make sure it is installed")?; - // Give it some time so hopefully the port is open. (100ms was not enough.) - thread::sleep(Duration::from_millis(200)); - - Ok(JodChild(josh)) -} |