//! Remotes (M4.10, owner decision D23): fetch, pull, push, clone and the //! list of configured remotes, through the user's git. //! //! - **Any server.** HTTPS and SSH remotes on any host (GitHub, GitLab, //! Forgejo, Bitbucket, a self-hosted server) and local paths work the //! same way, because git does the transport. `ext::` URLs are refused //! (`protocol.ext.allow=never`), so a URL never runs a command. //! - **Credentials stay with git.** HTTPS sign-in goes through git's //! credential helper, which may show its own window (Git Credential //! Manager); SSH uses the user's ssh, its agent and keys. Lutra never //! reads, stores or logs a credential or key, and never answers a //! prompt: terminal prompts are off ([`runner::Call::network`]), so a //! remote that needs a password nobody supplies fails with a message //! instead of hanging. //! - **Redaction.** Remote URLs ([`Remote`]) and every message have the //! userinfo of URLs replaced ([`runner::redact_urls`]). //! - **Clear messages.** A refusal keeps git's text and gets a short //! explanation first when its cause is known ([`explain`]): a rejected //! push, a sign-in that failed, an unknown SSH host key, an unreachable //! or missing repository, no upstream, conflicts after a pull. //! - **Progress and cancel.** Network calls run with `--progress`; each //! stderr line reaches the [`Network`] sink, and its cancel flag kills //! git and every process it started. use std::path::Path; use std::time::Duration; use crate::VcsError; use crate::repo::{Git, Repo}; use crate::runner::{self, Call, Cancel, ProgressSink, redact_urls}; /// Longest URL accepted. pub const MAX_URL_LEN: usize = 4096; /// Cap on config and push output read. const SMALL_CAP: usize = 1024 * 1024; /// Global options of every network call: no `ext::` transport. const NETWORK_OPTIONS: [&str; 2] = ["-c", "protocol.ext.allow=never"]; /// Progress and cancel of one network call. #[derive(Debug, Clone, Default)] pub struct Network { /// Receives git's progress lines. pub progress: Option, /// Stops the call when set. pub cancel: Option, } /// A configured remote. #[derive(Debug, Clone, PartialEq, Eq)] pub struct Remote { /// Its name, e.g. `origin`. pub name: String, /// `remote..url`, userinfo redacted. pub fetch_url: Option, /// `remote..pushurl`, userinfo redacted; `None` when pushes go /// to the fetch URL. pub push_url: Option, } /// How `git pull` joins the fetched commits. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum PullMode { /// `git pull --no-rebase`: a merge (fast-forward when possible). Merge, /// `git pull --rebase`: local commits are replayed on top. Rebase, } /// What [`Repo::push`] pushed. #[derive(Debug, Clone, PartialEq, Eq)] pub struct Pushed { /// The remote pushed to. pub remote: String, /// The local branch pushed. pub branch: String, /// The branch had no upstream and got one with this push. pub upstream_set: bool, } /// Checks a remote URL or path from the caller. /// /// # Errors /// [`VcsError::InvalidInput`] when it is blank, longer than /// [`MAX_URL_LEN`], starts with `-` (an option), holds a control /// character, or uses the `ext::` transport. pub fn check_url(url: &str) -> Result<(), VcsError> { let invalid = |why: &str| VcsError::InvalidInput(format!("invalid remote URL: {why}")); if url.trim().is_empty() { return Err(invalid("it is empty")); } if url.len() > MAX_URL_LEN { return Err(invalid("it is too long")); } if url.starts_with('-') { return Err(invalid("it starts with '-'")); } if url.chars().any(char::is_control) { return Err(invalid("it holds a control character")); } if url.trim().to_ascii_lowercase().starts_with("ext::") { return Err(invalid("the ext:: transport is not allowed")); } Ok(()) } /// Checks a remote name from the caller; git checks the rest /// (`valid_remote_name`). /// /// # Errors /// [`VcsError::InvalidName`] when it is empty, starts with `-`, or holds /// whitespace or a control character. pub fn check_remote_name(name: &str) -> Result<(), VcsError> { if name.is_empty() || name.len() > 255 || name.starts_with('-') || name.chars().any(|c| c.is_control() || c.is_whitespace()) { return Err(VcsError::InvalidName(name.to_string())); } Ok(()) } /// Known causes of a refusal, each with the text Lutra puts before git's /// message. Checked in order; the first that matches wins. const EXPLANATIONS: &[(&[&str], &str)] = &[ ( &[ "[rejected]", "non-fast-forward", "fetch first", "Updates were rejected", ], "The push was rejected: the remote branch has commits that your branch does not have. Pull first, then push again.", ), ( &[ "[remote rejected]", "pre-receive hook declined", "protected branch", ], "The server refused the push (for example a protected branch or a server-side hook).", ), ( &["Host key verification failed"], "The server's SSH host key is not known yet. Connect to it once from a terminal (for example ssh -T git@host) to check and accept the key, then try again.", ), ( &[ "Authentication failed", "could not read Username", "could not read Password", "terminal prompts disabled", "Permission denied (publickey", "Invalid username or password", "returned error: 401", "returned error: 403", "The requested URL returned error: 403", ], "Git could not sign in to the remote. Lutra leaves sign-in to git: set up its credential helper (for example sign in once with Git Credential Manager) or add your SSH key to the ssh agent, then try again.", ), ( &[ "Repository not found", "does not appear to be a git repository", "returned error: 404", "not found", ], "The remote repository was not found. Check the URL and that you have access to it.", ), ( &[ "Could not resolve host", "Failed to connect", "Connection timed out", "Connection refused", "Network is unreachable", "Could not read from remote repository", ], "The remote could not be reached. Check the URL and your network connection.", ), ( &[ "There is no tracking information", "no upstream", "has no upstream branch", ], "The current branch has no upstream branch. Push it once to set one.", ), ( &[ "would be overwritten", "You have unstaged changes", "Please commit or stash them", "cannot pull with rebase", ], "Local changes are in the way. Commit or stash them, then pull again.", ), ( &[ "CONFLICT", "Automatic merge failed", "could not apply", "Resolve all conflicts", ], "The pull stopped with conflicts. Resolve them, then commit the merge or continue the rebase.", ), ( &[ "Not possible to fast-forward", "Need to specify how to reconcile divergent branches", ], "Your branch and the remote branch have diverged. Choose merge or rebase for pulls in Settings › Git, then pull again.", ), ( &["already exists and is not an empty directory"], "The target folder exists and is not empty. Choose an empty or new folder.", ), ]; /// git's message with a short explanation first when its cause is known /// (see [`EXPLANATIONS`]); URLs are redacted. pub fn explain(message: &str) -> String { let message = redact_urls(message.trim()); let lower = message.to_lowercase(); let found = EXPLANATIONS.iter().find(|(needles, _)| { needles .iter() .any(|needle| lower.contains(&needle.to_lowercase())) }); match found { Some((_, why)) if message.is_empty() => (*why).to_string(), Some((_, why)) => format!("{why}\n\n{message}"), None => message, } } /// A progress line split into its phase and percentage: `Receiving /// objects: 45% (450/1000), 1.2 MiB` → (`Receiving objects`, 45). A /// `remote: ` prefix is dropped. Lines without a percentage give the /// whole line and `None`. pub fn parse_progress(line: &str) -> (String, Option) { let line = line.trim(); let line = line.strip_prefix("remote:").map_or(line, str::trim); if let Some((phase, rest)) = line.split_once(':') && let Some((number, _)) = rest.trim_start().split_once('%') && let Ok(percent) = number.trim().parse::() { return (phase.trim().to_string(), Some(percent.min(100))); } (line.to_string(), None) } /// Turns a failed network call's error into one with [`explain`]'s text. fn explained(error: VcsError) -> VcsError { match error { VcsError::Git(message) => VcsError::Git(explain(&message)), other => other, } } /// A network call in `cwd` with [`NETWORK_OPTIONS`] first and the /// progress sink and cancel flag of `net`. fn network_call<'a>(cwd: &'a Path, args: &[&str], net: &Network) -> Call<'a> { let mut all: Vec<&str> = NETWORK_OPTIONS.to_vec(); all.extend_from_slice(args); Call::network(cwd, &all) .progress(net.progress.clone()) .cancel(net.cancel.clone()) } /// Checks a network call's output, with explained errors. fn checked(output: Result) -> Result { output?.checked().map_err(explained) } impl Git { /// Clones `url` into `dir` (`git clone --progress`), with progress and /// cancel. /// /// # Parameters /// - `url`: any URL or path git can clone from; checked with /// [`check_url`]. /// - `dir`: the new work tree; absolute, its parent must exist, and it /// must be missing or an empty folder. /// /// When the clone fails or is cancelled, what it left in `dir` is /// removed again (the folder itself too, if the clone created it). /// /// # Errors /// [`VcsError::InvalidInput`] for a bad URL or folder, /// [`VcsError::Cancelled`], or git's refusal with [`explain`]'s text. pub fn clone_repo(&self, url: &str, dir: &Path, net: &Network) -> Result<(), VcsError> { check_url(url)?; let invalid = |why: &str| VcsError::InvalidInput(why.to_string()); if !dir.is_absolute() { return Err(invalid("the target folder must be an absolute path")); } let Some(parent) = dir.parent().filter(|p| p.is_dir()) else { return Err(invalid("the parent of the target folder does not exist")); }; let existed = dir.exists(); if existed { let empty = std::fs::read_dir(dir) .map(|mut entries| entries.next().is_none()) .unwrap_or(false); if !dir.is_dir() || !empty { return Err(invalid( "the target folder exists and is not empty; choose an empty or new folder", )); } } let Some(target) = dir.to_str() else { return Err(invalid("the target folder's path is not valid Unicode")); }; let result = checked(runner::run( self.executable(), network_call( parent, &["clone", "--progress", "--end-of-options", url, target], net, ), )); if result.is_err() { clean_up(dir, existed); } result.map(|_| ()) } } /// Removes what a failed clone left: the folder, or only its contents when /// it existed (empty) before. Failures are ignored after a few tries: the /// clone's error is what the caller reports, and a leftover folder is /// visible to the user. fn clean_up(dir: &Path, existed: bool) { for _ in 0..5 { let removed = if existed { std::fs::read_dir(dir).is_ok_and(|entries| { entries.flatten().all(|entry| { let path = entry.path(); if path.is_dir() { std::fs::remove_dir_all(&path).is_ok() } else { std::fs::remove_file(&path).is_ok() } }) }) } else { std::fs::remove_dir_all(dir).is_ok() || !dir.exists() }; if removed { return; } std::thread::sleep(Duration::from_millis(100)); } } /// Parses `git config -z --get-regexp '^remote\..*\.(url|pushurl)$'` /// output into remotes, in the order they are configured; the last value /// of a key wins, as in git. URLs are redacted. pub fn parse_remotes(bytes: &[u8]) -> Vec { let mut remotes: Vec = Vec::new(); for entry in bytes.split(|b| *b == 0) { let text = String::from_utf8_lossy(entry); let Some((key, value)) = text.split_once('\n') else { continue; }; let Some(rest) = key.strip_prefix("remote.") else { continue; }; let (name, push) = if let Some(name) = rest.strip_suffix(".pushurl") { (name, true) } else if let Some(name) = rest.strip_suffix(".url") { (name, false) } else { continue; }; if name.is_empty() || value.is_empty() { continue; } let index = match remotes.iter().position(|r| r.name == name) { Some(index) => index, None => { remotes.push(Remote { name: name.to_string(), fetch_url: None, push_url: None, }); remotes.len() - 1 } }; let Some(remote) = remotes.get_mut(index) else { continue; }; let value = Some(redact_urls(value)); if push { remote.push_url = value; } else { remote.fetch_url = value; } } remotes } /// The summaries of the refs `git push --porcelain` reports as rejected /// (lines starting with `!`), e.g. `[rejected] (fetch first)`. pub fn push_rejections(stdout: &[u8]) -> Vec { String::from_utf8_lossy(stdout) .lines() .filter_map(|line| { let rest = line.strip_prefix('!')?; let summary = rest.rsplit('\t').next().unwrap_or(rest).trim(); Some(summary.to_string()) }) .collect() } impl Repo { /// Runs a read in the work tree root. fn read(&self, args: &[&str]) -> Result { self.run(Call::read(self.root(), args, SMALL_CAP)) } /// One config value; `None` when it is not set. fn config(&self, key: &str) -> Result, VcsError> { let output = self.read(&["config", "--get", "--end-of-options", key])?; if !output.success() { return Ok(None); } let value = String::from_utf8_lossy(&output.stdout).trim().to_string(); Ok((!value.is_empty()).then_some(value)) } /// The configured remotes, in config order, URLs redacted. /// /// # Errors /// The runner's errors; no remote is an empty list. pub fn remotes(&self) -> Result, VcsError> { let output = self.read(&[ "config", "-z", "--get-regexp", r"^remote\..*\.(url|pushurl)$", ])?; if !output.success() { return Ok(Vec::new()); } Ok(parse_remotes(&output.stdout)) } /// Runs a remote-management call (`git remote …`), serialised with /// the other changes. fn manage(&self, args: &[&str]) -> Result<(), VcsError> { let _guard = self.mutation_guard(); self.run(Call::write(self.root(), args))?.checked()?; Ok(()) } /// Adds a remote (`git remote add`), without fetching. /// /// # Errors /// [`VcsError::InvalidName`], [`VcsError::InvalidInput`], or git's /// refusal (the remote exists, the name is not valid). pub fn remote_add(&self, name: &str, url: &str) -> Result<(), VcsError> { check_remote_name(name)?; check_url(url)?; self.manage(&["remote", "add", "--end-of-options", name, url]) } /// Changes a remote's URL (`git remote set-url`). /// /// # Errors /// As [`Repo::remote_add`]; an unknown remote is git's refusal. pub fn remote_set_url(&self, name: &str, url: &str) -> Result<(), VcsError> { check_remote_name(name)?; check_url(url)?; self.manage(&["remote", "set-url", "--end-of-options", name, url]) } /// Removes a remote with its remote-tracking branches and the /// upstream settings that named it (`git remote remove`). /// /// # Errors /// [`VcsError::InvalidName`], or git's refusal (no such remote). pub fn remote_remove(&self, name: &str) -> Result<(), VcsError> { check_remote_name(name)?; self.manage(&["remote", "remove", "--end-of-options", name]) } /// Fetches one remote, or every remote when `remote` is `None` /// (`git fetch --all`). /// /// # Errors /// [`VcsError::InvalidName`], [`VcsError::Cancelled`], or git's /// refusal with [`explain`]'s text. pub fn fetch(&self, remote: Option<&str>, net: &Network) -> Result<(), VcsError> { let args: Vec<&str> = match remote { Some(name) => { check_remote_name(name)?; vec!["fetch", "--progress", "--end-of-options", name] } None => vec!["fetch", "--progress", "--all"], }; let _guard = self.mutation_guard(); checked(self.run(network_call(self.root(), &args, net))).map(|_| ()) } /// Pulls the current branch's upstream. A merge takes git's default /// message without an editor (`--no-edit`). /// /// # Errors /// [`VcsError::Cancelled`], or git's refusal with [`explain`]'s text /// (no upstream, local changes in the way, conflicts). pub fn pull(&self, mode: PullMode, net: &Network) -> Result<(), VcsError> { let args: &[&str] = match mode { PullMode::Merge => &["pull", "--progress", "--no-rebase", "--no-edit"], PullMode::Rebase => &["pull", "--progress", "--rebase"], }; let _guard = self.mutation_guard(); checked(self.run(network_call(self.root(), args, net))).map(|_| ()) } /// Pushes the current branch. With an upstream it goes there /// (`branch..remote` and `.merge`); without one it goes to a /// branch of the same name on `remote` and becomes the upstream /// (`--set-upstream`). Without `remote` that is the only remote, else /// `origin`. Hooks (`pre-push`) run. /// /// # Errors /// [`VcsError::InvalidName`]; [`VcsError::Git`] when HEAD is detached, /// no remote fits, or git or the server refuses (with [`explain`]'s /// text, also when git's exit status hides a rejected ref); /// [`VcsError::Cancelled`]. pub fn push(&self, remote: Option<&str>, net: &Network) -> Result { if let Some(name) = remote { check_remote_name(name)?; } let head = self.head()?; let Some(branch) = head.branch.filter(|_| !head.detached) else { return Err(VcsError::Git( "HEAD is detached; check out a branch to push it.".into(), )); }; let upstream_remote = self .config(&format!("branch.{branch}.remote"))? .filter(|r| r != "."); let merge = self.config(&format!("branch.{branch}.merge"))?; let local_ref = format!("refs/heads/{branch}"); let (target, refspec, set_upstream) = match (upstream_remote, merge) { (Some(up), Some(merge)) if remote.is_none_or(|r| r == up) => { (up, format!("{local_ref}:{merge}"), false) } _ => { let target = match remote { Some(name) => name.to_string(), None => self.default_remote()?, }; (target, format!("{local_ref}:{local_ref}"), true) } }; let mut args = vec!["push", "--progress", "--porcelain"]; if set_upstream { args.push("--set-upstream"); } args.extend(["--end-of-options", &target, &refspec]); let _guard = self.mutation_guard(); let output = self.run(network_call(self.root(), &args, net))?; let rejected = push_rejections(&output.stdout); if !output.success() || !rejected.is_empty() { let mut message = output.message(); for line in rejected { if !message.contains(&line) { message.push('\n'); message.push_str(&line); } } return Err(VcsError::Git(explain(&message))); } Ok(Pushed { remote: target, branch, upstream_set: set_upstream, }) } /// The remote a branch without upstream is pushed to: the only remote, /// else `origin`. fn default_remote(&self) -> Result { let remotes = self.remotes()?; match remotes.as_slice() { [only] => Ok(only.name.clone()), [] => Err(VcsError::Git( "This repository has no remote to push to. Add one in Git › Manage Remotes…".into(), )), many if many.iter().any(|r| r.name == "origin") => Ok("origin".into()), _ => Err(VcsError::Git( "This repository has several remotes and none is named origin; choose the remote to push to.".into(), )), } } } #[cfg(test)] mod tests { #![allow(clippy::unwrap_used)] use super::*; #[test] fn urls_are_checked() { assert!(check_url("https://github.com/a/b.git").is_ok()); assert!(check_url("git@github.com:a/b.git").is_ok()); assert!(check_url("C:\\repos\\x.git").is_ok()); assert!(check_url("").is_err()); assert!(check_url(" ").is_err()); assert!(check_url("--upload-pack=evil").is_err()); assert!(check_url("https://a\nb").is_err()); assert!(check_url("ext::sh -c touch% /tmp/x").is_err()); assert!(check_url(&"a".repeat(MAX_URL_LEN + 1)).is_err()); } #[test] fn remote_names_are_checked() { assert!(check_remote_name("origin").is_ok()); assert!(check_remote_name("my.fork/x").is_ok()); assert!(check_remote_name("").is_err()); assert!(check_remote_name("-x").is_err()); assert!(check_remote_name("a b").is_err()); } #[test] fn explanations_come_before_git_message() { let text = explain( " ! [rejected] main -> main (fetch first)\nerror: failed to push some refs to 'https://u:p@h/r'", ); assert!(text.starts_with("The push was rejected")); assert!(text.contains("https://***@h/r")); assert!(!text.contains("u:p")); assert!( explain( "fatal: could not read Username for 'https://github.com': terminal prompts disabled" ) .starts_with("Git could not sign in") ); assert!( explain("Host key verification failed.\nfatal: Could not read from remote repository.") .starts_with("The server's SSH host key") ); assert!( explain("fatal: '/x' does not appear to be a git repository") .starts_with("The remote repository was not found") ); assert!( explain("There is no tracking information for the current branch.") .starts_with("The current branch has no upstream") ); assert_eq!(explain("something else"), "something else"); } #[test] fn progress_lines_give_phase_and_percent() { assert_eq!( parse_progress("Receiving objects: 45% (450/1000), 1.20 MiB | 2.00 MiB/s"), ("Receiving objects".into(), Some(45)) ); assert_eq!( parse_progress("remote: Counting objects: 100% (3/3), done."), ("Counting objects".into(), Some(100)) ); assert_eq!( parse_progress("Cloning into 'x'..."), ("Cloning into 'x'...".into(), None) ); assert_eq!(parse_progress("a: 999% b"), ("a: 999% b".into(), None)); } #[test] fn remotes_parse_in_order_with_redacted_urls() { let bytes = b"remote.origin.url\nhttps://me:tok@github.com/a/b.git\0remote.my.fork.url\ngit@h:x.git\0remote.origin.pushurl\nssh://git@h/b\0remote.bad.fetch\nx\0junk\0"; let remotes = parse_remotes(bytes); assert_eq!( remotes, [ Remote { name: "origin".into(), fetch_url: Some("https://***@github.com/a/b.git".into()), push_url: Some("ssh://***@h/b".into()), }, Remote { name: "my.fork".into(), fetch_url: Some("git@h:x.git".into()), push_url: None, }, ] ); } #[test] fn push_rejections_are_read_from_porcelain() { let out = b"To /tmp/r.git\n!\trefs/heads/main:refs/heads/main\t[rejected] (fetch first)\nDone\n"; assert_eq!(push_rejections(out), ["[rejected] (fetch first)"]); assert!( push_rejections(b"To x\n=\trefs/heads/a:refs/heads/a\t[up to date]\nDone\n").is_empty() ); } proptest::proptest! { #[test] fn parsers_take_any_bytes(bytes in proptest::collection::vec(proptest::num::u8::ANY, 0..300)) { let _ = parse_remotes(&bytes); let _ = push_rejections(&bytes); let text = String::from_utf8_lossy(&bytes); let _ = parse_progress(&text); let _ = explain(&text); } } }