//! Jobs: test a connection, upload or download project paths. //! //! A job first plans (walks the local folders or lists the remote ones and //! maps every file), then connects once and transfers file by file. Its //! [`Monitor`] holds the progress and the new log lines; the engine reads //! it from another thread. A failing file is logged and the job goes on; a //! lost connection, a refused sign-in or a cancel ends it. //! //! Folder walks skip symbolic links to folders and the version-control //! folders `.git`, `.svn` and `.hg`, which are never deployed. use std::collections::{BTreeMap, BTreeSet}; use std::fs::File; use std::path::Path; use std::sync::{Arc, Mutex, PoisonError}; use serde::Serialize; use crate::config::{Server, ServerKind, io_error}; use crate::error::{DeployError, Result}; use crate::mapping::{is_excluded, join, normalize_relative, remote_path}; use crate::remote::{Cancel, Folder, Remote}; use crate::secrets::Secret; /// Most files one job plans. pub const MAX_FILES: usize = 200_000; /// Deepest folder nesting a job follows. const MAX_DEPTH: usize = 64; /// Folders never deployed. const VCS_DIRS: [&str; 3] = [".git", ".svn", ".hg"]; /// Severity of a log line. #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] #[serde(rename_all = "lowercase")] pub enum Level { /// Something worked. Info, /// Something was skipped. Warning, /// Something failed. Error, } /// One line of the deployment log. #[derive(Debug, Clone, PartialEq, Eq, Serialize)] pub struct LogLine { /// Severity. pub level: Level, /// The text; never contains a secret. pub message: String, } /// Progress of a running job. #[derive(Debug, Clone, Default, PartialEq, Eq, Serialize)] #[serde(rename_all = "camelCase")] pub struct Progress { /// `connecting`, `planning`, `comparing`, `transferring`. pub phase: String, /// The file being transferred, project-relative. #[serde(skip_serializing_if = "Option::is_none")] pub file: Option, /// Files finished, successful or not. pub files_done: usize, /// Files planned. pub files_total: usize, /// Bytes transferred. pub bytes_done: u64, /// Bytes planned (uploads only; 0 when unknown). pub bytes_total: u64, } /// What a job shares with its observer. #[derive(Debug, Default)] struct Shared { /// The progress. progress: Progress, /// Log lines not taken yet. lines: Vec, } /// A job's progress and log, readable from another thread. Clones share /// the state. #[derive(Debug, Clone, Default)] pub struct Monitor(Arc>); impl Monitor { /// A fresh monitor. pub fn new() -> Self { Self::default() } /// The current progress. pub fn progress(&self) -> Progress { self.0 .lock() .unwrap_or_else(PoisonError::into_inner) .progress .clone() } /// Takes the log lines written since the last call. pub fn take_lines(&self) -> Vec { std::mem::take(&mut self.0.lock().unwrap_or_else(PoisonError::into_inner).lines) } /// Changes the progress. fn update(&self, change: impl FnOnce(&mut Progress)) { change( &mut self .0 .lock() .unwrap_or_else(PoisonError::into_inner) .progress, ); } /// Adds a log line. pub(crate) fn log(&self, level: Level, message: String) { self.0 .lock() .unwrap_or_else(PoisonError::into_inner) .lines .push(LogLine { level, message }); } } /// Everything a job needs. #[derive(Debug, Clone)] pub struct Job { /// The server. pub server: Server, /// The project root on disk. pub project_root: std::path::PathBuf, /// The password or passphrase from the keychain, when the server /// needs one and it is stored. pub secret: Option, /// Lutra's known-hosts file. pub known_hosts: std::path::PathBuf, /// Stops the job. pub cancel: Cancel, /// Progress and log. pub monitor: Monitor, } /// What a finished upload or download did. #[derive(Debug, Clone, Default, PartialEq, Eq, Serialize)] #[serde(rename_all = "camelCase")] pub struct Summary { /// Files transferred. pub files: usize, /// Bytes transferred. pub bytes: u64, /// Files skipped: excluded, not mapped, or missing on the server. pub skipped: usize, /// Files that failed. pub failed: usize, /// Files deleted on the server (sync only). pub deleted: usize, } /// One planned file. #[derive(Debug, Clone)] struct Planned { /// Project-relative path. local: String, /// Remote path. remote: String, /// Size, when known. size: u64, } /// Connects to the job's server. /// /// # Errors /// What the transport's `connect` returns, with the secret redacted. pub fn connect(job: &Job) -> Result> { job.monitor.update(|p| p.phase = "connecting".into()); let secret = job.secret.as_ref(); let remote: Result> = match job.server.kind { ServerKind::Sftp => { crate::sftp::Sftp::connect(&job.server, secret, &job.known_hosts, &job.cancel) .map(|s| Box::new(s) as Box) } ServerKind::Ftp | ServerKind::Ftps => { crate::ftp::Ftp::connect(&job.server, secret, &job.cancel) .map(|s| Box::new(s) as Box) } ServerKind::Folder => { Folder::open(Path::new(&job.server.root)).map(|s| Box::new(s) as Box) } }; remote.map_err(|e| e.redacted(secret.map(Secret::expose))) } /// Connects, signs in and checks the root folder. Returns a sentence for /// the user. /// /// # Errors /// What [`connect`] returns. pub fn test_connection(job: &Job) -> Result { let mut remote = connect(job)?; let root = if job.server.kind == ServerKind::Folder { String::new() } else { job.server.root.clone() }; let kind = remote .kind(&root) .map_err(|e| e.redacted(job.secret.as_ref().map(Secret::expose))); remote.close(); let kind = kind?; let name = &job.server.name; let shown = display_root(&root); let message = match kind { _ if job.server.kind == ServerKind::Folder => { format!("Connected to the folder {}.", job.server.root) } Some(true) => format!("Connected to {name}; the root folder {shown} exists."), Some(false) => { format!("Connected to {name}, but the root {shown} is a file, not a folder.") } None => format!( "Connected to {name}; the root folder {shown} does not exist yet and is created by the first upload." ), }; job.monitor.log(Level::Info, message.clone()); Ok(message) } /// The root as shown to the user. fn display_root(root: &str) -> &str { if root.is_empty() { "(login folder)" } else { root } } /// Adds the files under the project-relative `rel` to `plan`. fn walk_local( job: &Job, rel: &str, depth: usize, plan: &mut Vec, skipped: &mut usize, ) -> Result<()> { job.cancel.check()?; let path = if rel.is_empty() { job.project_root.clone() } else { job.project_root.join(rel) }; let meta = std::fs::symlink_metadata(&path).map_err(|e| io_error(&path, &e))?; let is_dir = meta.is_dir() || (meta.file_type().is_symlink() && depth == 0 && path.is_dir()); if !is_dir { let size = std::fs::metadata(&path) .map_err(|e| io_error(&path, &e))? .len(); match remote_path(&job.server, rel) { Some(remote) if !rel.is_empty() => plan.push(Planned { local: rel.to_string(), remote, size, }), _ => { *skipped += 1; job.monitor.log( Level::Warning, format!("{rel}: skipped (excluded or not mapped)"), ); } } if plan.len() > MAX_FILES { return Err(DeployError::Invalid(format!( "more than {MAX_FILES} files; upload a smaller folder" ))); } return Ok(()); } if depth > MAX_DEPTH || (!rel.is_empty() && is_excluded(&job.server, rel)) { return Ok(()); } let mut names: Vec = std::fs::read_dir(&path) .map_err(|e| io_error(&path, &e))? .filter_map(|e| e.ok()) .filter(|e| { e.file_type() .map(|t| !(t.is_symlink() && e.path().is_dir())) .unwrap_or(false) }) .map(|e| e.file_name().to_string_lossy().into_owned()) .filter(|n| !VCS_DIRS.contains(&n.as_str())) .collect(); names.sort(); for name in names { let child = if rel.is_empty() { name } else { format!("{rel}/{name}") }; walk_local(job, &child, depth + 1, plan, skipped)?; } Ok(()) } /// The parent folder of a remote path; empty for a top-level one. fn parent(remote: &str) -> &str { match remote.rfind('/') { Some(0) => "/", Some(i) => remote.get(..i).unwrap_or_default(), None => "", } } /// Whether an error ends the whole job instead of one file. fn is_fatal(e: &DeployError) -> bool { !matches!(e, DeployError::Transfer(_)) } /// Uploads the project-relative `paths` (files or folders). /// /// # Errors /// [`DeployError::Invalid`] for a path outside the project, otherwise /// what [`connect`] returns, or [`DeployError::Cancelled`]. Failing files /// are counted in [`Summary::failed`] instead. pub fn upload(job: &Job, paths: &[String]) -> Result { let secret = job.secret.as_ref().map(Secret::expose); job.monitor.update(|p| p.phase = "planning".into()); let mut plan = Vec::new(); let mut summary = Summary::default(); for path in paths { let rel = normalize_relative(path) .ok_or_else(|| DeployError::Invalid(format!("{path:?} is not inside the project")))?; walk_local(job, &rel, 0, &mut plan, &mut summary.skipped)?; } let total: u64 = plan.iter().map(|p| p.size).sum(); job.monitor.update(|p| { p.files_total = plan.len(); p.bytes_total = total; }); if plan.is_empty() { job.monitor.log(Level::Warning, "nothing to upload".into()); return Ok(summary); } let mut remote = connect(job)?; job.monitor.update(|p| p.phase = "transferring".into()); let mut made: BTreeSet = BTreeSet::new(); let result = (|| { for file in &plan { job.cancel.check()?; job.monitor.update(|p| p.file = Some(file.local.clone())); let outcome = upload_one(job, remote.as_mut(), file, &mut made).map_err(|e| e.redacted(secret)); job.monitor.update(|p| p.files_done += 1); match outcome { Ok(()) => { summary.files += 1; summary.bytes += file.size; job.monitor .log(Level::Info, format!("{} → {}", file.local, file.remote)); } Err(e) if is_fatal(&e) => return Err(e), Err(e) => { summary.failed += 1; job.monitor .log(Level::Error, format!("{}: {e}", file.local)); } } } Ok(()) })(); remote.close(); result.map(|()| summary) } /// Uploads one file, creating its remote folder first. fn upload_one( job: &Job, remote: &mut dyn Remote, file: &Planned, made: &mut BTreeSet, ) -> Result<()> { let dir = parent(&file.remote).to_string(); if !dir.is_empty() && dir != "/" && !made.contains(&dir) { remote.mkdir_all(&dir)?; made.insert(dir); } let path = job.project_root.join(&file.local); let mut data = File::open(&path).map_err(|e| io_error(&path, &e))?; let monitor = job.monitor.clone(); let cancel = job.cancel.clone(); remote.put(&file.remote, &mut data, &mut |n| { monitor.update(|p| p.bytes_done = p.bytes_done.saturating_add(n)); cancel.check() }) } /// Adds the remote files under the project-relative `rel` to `plan`. fn walk_remote( job: &Job, remote: &mut dyn Remote, rel: &str, depth: usize, plan: &mut Vec, skipped: &mut usize, ) -> Result<()> { job.cancel.check()?; let Some(path) = remote_path(&job.server, rel) else { *skipped += 1; job.monitor.log( Level::Warning, format!("{rel}: skipped (excluded or not mapped)"), ); return Ok(()); }; match remote.kind(&path)? { None => { *skipped += 1; job.monitor .log(Level::Warning, format!("{rel}: not on the server ({path})")); } Some(false) if !rel.is_empty() => plan.push(Planned { local: rel.to_string(), remote: path, size: 0, }), Some(false) => { *skipped += 1; } Some(true) => walk_remote_tree(job, remote, rel, &path, depth, plan)?, } Ok(()) } /// Downloads the project-relative `paths` (files or folders) from the /// server into the project, replacing local files. /// /// # Errors /// Like [`upload`]. pub fn download(job: &Job, paths: &[String]) -> Result { let secret = job.secret.as_ref().map(Secret::expose); let mut rels = Vec::new(); for path in paths { rels.push( normalize_relative(path).ok_or_else(|| { DeployError::Invalid(format!("{path:?} is not inside the project")) })?, ); } let mut remote = connect(job)?; let mut summary = Summary::default(); let result = (|| { job.monitor.update(|p| p.phase = "planning".into()); let mut plan = Vec::new(); for rel in &rels { walk_remote( job, remote.as_mut(), rel, 0, &mut plan, &mut summary.skipped, ) .map_err(|e| e.redacted(secret))?; } job.monitor.update(|p| { p.files_total = plan.len(); p.phase = "transferring".into(); }); for file in &plan { job.cancel.check()?; job.monitor.update(|p| p.file = Some(file.local.clone())); let outcome = download_one(job, remote.as_mut(), file).map_err(|e| e.redacted(secret)); job.monitor.update(|p| p.files_done += 1); match outcome { Ok(bytes) => { summary.files += 1; summary.bytes += bytes; job.monitor .log(Level::Info, format!("{} ← {}", file.local, file.remote)); } Err(e) if is_fatal(&e) => return Err(e), Err(e) => { summary.failed += 1; job.monitor .log(Level::Error, format!("{}: {e}", file.local)); } } } Ok(()) })(); remote.close(); result.map(|()| summary) } /// Downloads one file through a temporary file next to it. Returns its /// size. A temporary file that cannot be removed after a failure is left; /// the next download of the same file replaces it. fn download_one(job: &Job, remote: &mut dyn Remote, file: &Planned) -> Result { let target = job.project_root.join(&file.local); if let Some(dir) = target.parent() { std::fs::create_dir_all(dir).map_err(|e| io_error(dir, &e))?; } let temp = target.with_extension("lutra-download"); let monitor = job.monitor.clone(); let cancel = job.cancel.clone(); let mut bytes = 0u64; let result = (|| { let mut out = File::create(&temp).map_err(|e| io_error(&temp, &e))?; remote.get(&file.remote, &mut out, &mut |n| { bytes = bytes.saturating_add(n); monitor.update(|p| p.bytes_done = p.bytes_done.saturating_add(n)); cancel.check() })?; drop(out); std::fs::rename(&temp, &target).map_err(|e| io_error(&target, &e)) })(); if result.is_err() { let _ = std::fs::remove_file(&temp); } result.map(|()| bytes) } /// Adds the files below the remote folder `path` (the mapped folder of the /// project-relative `rel`) to `plan`, from its listings: sizes come from /// the server, excluded and unmapped entries are left out, and a child /// that maps elsewhere is not followed here. fn walk_remote_tree( job: &Job, remote: &mut dyn Remote, rel: &str, path: &str, depth: usize, plan: &mut Vec, ) -> Result<()> { if depth > MAX_DEPTH { return Ok(()); } for entry in remote.list(path)? { job.cancel.check()?; if VCS_DIRS.contains(&entry.name.as_str()) { continue; } let child = if rel.is_empty() { entry.name.clone() } else { format!("{rel}/{}", entry.name) }; let child_path = join(&[path, &entry.name]); if remote_path(&job.server, &child).as_deref() != Some(child_path.as_str()) { continue; } if entry.is_dir { walk_remote_tree(job, remote, &child, &child_path, depth + 1, plan)?; } else { plan.push(Planned { local: child, remote: child_path, size: entry.size.unwrap_or(0), }); } if plan.len() > MAX_FILES { return Err(DeployError::Invalid(format!( "more than {MAX_FILES} files; choose a smaller folder" ))); } } Ok(()) } /// Largest file whose contents [`compare`] reads to tell equal sizes /// apart; larger files of equal size count as equal. pub const MAX_COMPARE: u64 = 8 * 1024 * 1024; /// How a file differs between the project and the server. #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] #[serde(rename_all = "camelCase")] pub enum DiffStatus { /// Only in the project. LocalOnly, /// Only on the server. RemoteOnly, /// On both, with different contents. Different, } /// One differing file of [`compare`]. #[derive(Debug, Clone, PartialEq, Eq, Serialize)] #[serde(rename_all = "camelCase")] pub struct Difference { /// Project-relative path. pub path: String, /// How it differs. pub status: DiffStatus, /// Its size in the project. #[serde(skip_serializing_if = "Option::is_none")] pub local_size: Option, /// Its size on the server. #[serde(skip_serializing_if = "Option::is_none")] pub remote_size: Option, } /// Whether the local `file` and its remote counterpart hold the same /// bytes; callers check that both sizes match first. fn same_contents(job: &Job, remote: &mut dyn Remote, file: &Planned) -> Result { let path = job.project_root.join(&file.local); let local = std::fs::read(&path).map_err(|e| io_error(&path, &e))?; let mut theirs = Vec::with_capacity(local.len()); let cancel = job.cancel.clone(); remote.get(&file.remote, &mut theirs, &mut |_| cancel.check())?; Ok(local == theirs) } /// Compares the project-relative `paths` (files or folders) with the /// server: files only in the project, only on the server, or with /// different contents (sizes first; equal sizes up to [`MAX_COMPARE`] by /// contents). Excluded and unmapped files are left out. Sorted by path. /// /// # Errors /// [`DeployError::Invalid`] for a path outside the project, otherwise what /// the connection or the listing returns. pub fn compare(job: &Job, paths: &[String]) -> Result> { let secret = job.secret.as_ref().map(Secret::expose); job.monitor.update(|p| p.phase = "planning".into()); let mut local = Vec::new(); let mut skipped = 0; let mut rels = Vec::new(); for path in paths { let rel = normalize_relative(path) .ok_or_else(|| DeployError::Invalid(format!("{path:?} is not inside the project")))?; if job.project_root.join(&rel).exists() { walk_local(job, &rel, 0, &mut local, &mut skipped)?; } rels.push(rel); } let mut remote = connect(job)?; let result = (|| { let mut theirs = Vec::new(); for rel in &rels { let mut ignored = 0; walk_remote(job, remote.as_mut(), rel, 0, &mut theirs, &mut ignored)?; } let mut by_path: BTreeMap = BTreeMap::new(); for file in &local { by_path.insert( file.local.clone(), Difference { path: file.local.clone(), status: DiffStatus::LocalOnly, local_size: Some(file.size), remote_size: None, }, ); } job.monitor.update(|p| { p.phase = "comparing".into(); p.files_total = theirs.len(); }); for file in &theirs { job.cancel.check()?; job.monitor.update(|p| p.file = Some(file.local.clone())); match local.iter().find(|l| l.local == file.local) { None => { by_path.insert( file.local.clone(), Difference { path: file.local.clone(), status: DiffStatus::RemoteOnly, local_size: None, remote_size: Some(file.size), }, ); } Some(mine) => { let differs = mine.size != file.size || (mine.size <= MAX_COMPARE && !same_contents(job, remote.as_mut(), mine)?); if differs { by_path.insert( file.local.clone(), Difference { path: file.local.clone(), status: DiffStatus::Different, local_size: Some(mine.size), remote_size: Some(file.size), }, ); } else { by_path.remove(&file.local); } } } job.monitor.update(|p| p.files_done += 1); } Ok(by_path.into_values().collect::>()) })(); remote.close(); let found = result.map_err(|e: DeployError| e.redacted(secret))?; job.monitor.log( Level::Info, format!("compared: {} differing files", found.len()), ); Ok(found) } /// Uploads the project-relative files `upload` and deletes the files /// `delete` on the server, in one connection. Callers show the plan first /// and pass deletions only after the user confirmed them; the engine /// refuses deletions without that confirmation. /// /// # Errors /// Like [`upload`]; a failing file is counted in [`Summary::failed`]. pub fn sync(job: &Job, upload: &[String], delete: &[String]) -> Result { let secret = job.secret.as_ref().map(Secret::expose); let mut plan = Vec::new(); let mut summary = Summary::default(); for path in upload { let rel = normalize_relative(path) .ok_or_else(|| DeployError::Invalid(format!("{path:?} is not inside the project")))?; walk_local(job, &rel, 0, &mut plan, &mut summary.skipped)?; } let mut doomed = Vec::new(); for path in delete { let rel = normalize_relative(path) .filter(|r| !r.is_empty()) .ok_or_else(|| DeployError::Invalid(format!("{path:?} is not a project file")))?; let remote = remote_path(&job.server, &rel) .ok_or_else(|| DeployError::Invalid(format!("{rel} is excluded or not mapped")))?; doomed.push((rel, remote)); } job.monitor.update(|p| { p.files_total = plan.len() + doomed.len(); p.bytes_total = plan.iter().map(|f| f.size).sum(); }); let mut remote = connect(job)?; job.monitor.update(|p| p.phase = "transferring".into()); let mut made: BTreeSet = BTreeSet::new(); let result = (|| { for file in &plan { job.cancel.check()?; job.monitor.update(|p| p.file = Some(file.local.clone())); let outcome = upload_one(job, remote.as_mut(), file, &mut made).map_err(|e| e.redacted(secret)); job.monitor.update(|p| p.files_done += 1); match outcome { Ok(()) => { summary.files += 1; summary.bytes += file.size; job.monitor .log(Level::Info, format!("{} → {}", file.local, file.remote)); } Err(e) if is_fatal(&e) => return Err(e), Err(e) => { summary.failed += 1; job.monitor .log(Level::Error, format!("{}: {e}", file.local)); } } } for (rel, path) in &doomed { job.cancel.check()?; job.monitor.update(|p| p.file = Some(rel.clone())); let outcome = remote.remove_file(path).map_err(|e| e.redacted(secret)); job.monitor.update(|p| p.files_done += 1); match outcome { Ok(()) => { summary.deleted += 1; job.monitor .log(Level::Warning, format!("deleted {path} on the server")); } Err(e) if is_fatal(&e) => return Err(e), Err(e) => { summary.failed += 1; job.monitor.log(Level::Error, format!("{rel}: {e}")); } } } Ok(()) })(); remote.close(); result.map(|()| summary) } #[cfg(test)] mod tests { use super::*; use crate::config::{AuthKind, Mapping}; fn folder_job(project: &Path, target: &Path) -> Job { Job { server: Server { id: "f".into(), name: "Mounted".into(), kind: ServerKind::Folder, host: String::new(), port: None, user: String::new(), root: target.display().to_string(), auth: AuthKind::Password, key_path: None, mappings: vec![Mapping { local: String::new(), remote: "site".into(), }], excluded: vec!["*.log".into()], upload_on_save: false, }, project_root: project.to_path_buf(), secret: None, known_hosts: project.join("kh"), cancel: Cancel::new(), monitor: Monitor::new(), } } #[test] fn parents() { assert_eq!(parent("/a/b"), "/a"); assert_eq!(parent("/a"), "/"); assert_eq!(parent("a/b"), "a"); assert_eq!(parent("a"), ""); } #[test] fn upload_and_download_through_a_folder() { let project = tempfile::tempdir().unwrap(); let target = tempfile::tempdir().unwrap(); std::fs::create_dir_all(project.path().join("src/sub")).unwrap(); std::fs::create_dir_all(project.path().join(".git")).unwrap(); std::fs::write(project.path().join("src/a.php"), "