Remote trait gains remove_file, remove_dir and rename (SFTP, FTP, folder; test servers too). browse: list, read as text (binary and size caps), write, rename, mkdir, recursive delete that never takes the root. job: compare (local only, server only, different by size then contents) and sync (uploads plus explicitly listed deletions). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
871 lines
29 KiB
Rust
871 lines
29 KiB
Rust
//! Jobs: test a connection, upload or download project paths.
|
|
//!
|
|
//! A job first plans (walks the local folders or lists the remote ones and
|
|
//! maps every file), then connects once and transfers file by file. Its
|
|
//! [`Monitor`] holds the progress and the new log lines; the engine reads
|
|
//! it from another thread. A failing file is logged and the job goes on; a
|
|
//! lost connection, a refused sign-in or a cancel ends it.
|
|
//!
|
|
//! Folder walks skip symbolic links to folders and the version-control
|
|
//! folders `.git`, `.svn` and `.hg`, which are never deployed.
|
|
|
|
use std::collections::{BTreeMap, BTreeSet};
|
|
use std::fs::File;
|
|
use std::path::Path;
|
|
use std::sync::{Arc, Mutex, PoisonError};
|
|
|
|
use serde::Serialize;
|
|
|
|
use crate::config::{Server, ServerKind, io_error};
|
|
use crate::error::{DeployError, Result};
|
|
use crate::mapping::{is_excluded, join, normalize_relative, remote_path};
|
|
use crate::remote::{Cancel, Folder, Remote};
|
|
use crate::secrets::Secret;
|
|
|
|
/// Most files one job plans.
|
|
pub const MAX_FILES: usize = 200_000;
|
|
|
|
/// Deepest folder nesting a job follows.
|
|
const MAX_DEPTH: usize = 64;
|
|
|
|
/// Folders never deployed.
|
|
const VCS_DIRS: [&str; 3] = [".git", ".svn", ".hg"];
|
|
|
|
/// Severity of a log line.
|
|
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
|
|
#[serde(rename_all = "lowercase")]
|
|
pub enum Level {
|
|
/// Something worked.
|
|
Info,
|
|
/// Something was skipped.
|
|
Warning,
|
|
/// Something failed.
|
|
Error,
|
|
}
|
|
|
|
/// One line of the deployment log.
|
|
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
|
|
pub struct LogLine {
|
|
/// Severity.
|
|
pub level: Level,
|
|
/// The text; never contains a secret.
|
|
pub message: String,
|
|
}
|
|
|
|
/// Progress of a running job.
|
|
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize)]
|
|
#[serde(rename_all = "camelCase")]
|
|
pub struct Progress {
|
|
/// `connecting`, `planning`, `comparing`, `transferring`.
|
|
pub phase: String,
|
|
/// The file being transferred, project-relative.
|
|
#[serde(skip_serializing_if = "Option::is_none")]
|
|
pub file: Option<String>,
|
|
/// Files finished, successful or not.
|
|
pub files_done: usize,
|
|
/// Files planned.
|
|
pub files_total: usize,
|
|
/// Bytes transferred.
|
|
pub bytes_done: u64,
|
|
/// Bytes planned (uploads only; 0 when unknown).
|
|
pub bytes_total: u64,
|
|
}
|
|
|
|
/// What a job shares with its observer.
|
|
#[derive(Debug, Default)]
|
|
struct Shared {
|
|
/// The progress.
|
|
progress: Progress,
|
|
/// Log lines not taken yet.
|
|
lines: Vec<LogLine>,
|
|
}
|
|
|
|
/// A job's progress and log, readable from another thread. Clones share
|
|
/// the state.
|
|
#[derive(Debug, Clone, Default)]
|
|
pub struct Monitor(Arc<Mutex<Shared>>);
|
|
|
|
impl Monitor {
|
|
/// A fresh monitor.
|
|
pub fn new() -> Self {
|
|
Self::default()
|
|
}
|
|
|
|
/// The current progress.
|
|
pub fn progress(&self) -> Progress {
|
|
self.0
|
|
.lock()
|
|
.unwrap_or_else(PoisonError::into_inner)
|
|
.progress
|
|
.clone()
|
|
}
|
|
|
|
/// Takes the log lines written since the last call.
|
|
pub fn take_lines(&self) -> Vec<LogLine> {
|
|
std::mem::take(&mut self.0.lock().unwrap_or_else(PoisonError::into_inner).lines)
|
|
}
|
|
|
|
/// Changes the progress.
|
|
fn update(&self, change: impl FnOnce(&mut Progress)) {
|
|
change(
|
|
&mut self
|
|
.0
|
|
.lock()
|
|
.unwrap_or_else(PoisonError::into_inner)
|
|
.progress,
|
|
);
|
|
}
|
|
|
|
/// Adds a log line.
|
|
pub(crate) fn log(&self, level: Level, message: String) {
|
|
self.0
|
|
.lock()
|
|
.unwrap_or_else(PoisonError::into_inner)
|
|
.lines
|
|
.push(LogLine { level, message });
|
|
}
|
|
}
|
|
|
|
/// Everything a job needs.
|
|
#[derive(Debug, Clone)]
|
|
pub struct Job {
|
|
/// The server.
|
|
pub server: Server,
|
|
/// The project root on disk.
|
|
pub project_root: std::path::PathBuf,
|
|
/// The password or passphrase from the keychain, when the server
|
|
/// needs one and it is stored.
|
|
pub secret: Option<Secret>,
|
|
/// Lutra's known-hosts file.
|
|
pub known_hosts: std::path::PathBuf,
|
|
/// Stops the job.
|
|
pub cancel: Cancel,
|
|
/// Progress and log.
|
|
pub monitor: Monitor,
|
|
}
|
|
|
|
/// What a finished upload or download did.
|
|
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize)]
|
|
#[serde(rename_all = "camelCase")]
|
|
pub struct Summary {
|
|
/// Files transferred.
|
|
pub files: usize,
|
|
/// Bytes transferred.
|
|
pub bytes: u64,
|
|
/// Files skipped: excluded, not mapped, or missing on the server.
|
|
pub skipped: usize,
|
|
/// Files that failed.
|
|
pub failed: usize,
|
|
/// Files deleted on the server (sync only).
|
|
pub deleted: usize,
|
|
}
|
|
|
|
/// One planned file.
|
|
#[derive(Debug, Clone)]
|
|
struct Planned {
|
|
/// Project-relative path.
|
|
local: String,
|
|
/// Remote path.
|
|
remote: String,
|
|
/// Size, when known.
|
|
size: u64,
|
|
}
|
|
|
|
/// Connects to the job's server.
|
|
///
|
|
/// # Errors
|
|
/// What the transport's `connect` returns, with the secret redacted.
|
|
pub fn connect(job: &Job) -> Result<Box<dyn Remote>> {
|
|
job.monitor.update(|p| p.phase = "connecting".into());
|
|
let secret = job.secret.as_ref();
|
|
let remote: Result<Box<dyn Remote>> = match job.server.kind {
|
|
ServerKind::Sftp => {
|
|
crate::sftp::Sftp::connect(&job.server, secret, &job.known_hosts, &job.cancel)
|
|
.map(|s| Box::new(s) as Box<dyn Remote>)
|
|
}
|
|
ServerKind::Ftp | ServerKind::Ftps => {
|
|
crate::ftp::Ftp::connect(&job.server, secret, &job.cancel)
|
|
.map(|s| Box::new(s) as Box<dyn Remote>)
|
|
}
|
|
ServerKind::Folder => {
|
|
Folder::open(Path::new(&job.server.root)).map(|s| Box::new(s) as Box<dyn Remote>)
|
|
}
|
|
};
|
|
remote.map_err(|e| e.redacted(secret.map(Secret::expose)))
|
|
}
|
|
|
|
/// Connects, signs in and checks the root folder. Returns a sentence for
|
|
/// the user.
|
|
///
|
|
/// # Errors
|
|
/// What [`connect`] returns.
|
|
pub fn test_connection(job: &Job) -> Result<String> {
|
|
let mut remote = connect(job)?;
|
|
let root = if job.server.kind == ServerKind::Folder {
|
|
String::new()
|
|
} else {
|
|
job.server.root.clone()
|
|
};
|
|
let kind = remote
|
|
.kind(&root)
|
|
.map_err(|e| e.redacted(job.secret.as_ref().map(Secret::expose)));
|
|
remote.close();
|
|
let kind = kind?;
|
|
let name = &job.server.name;
|
|
let shown = display_root(&root);
|
|
let message = match kind {
|
|
_ if job.server.kind == ServerKind::Folder => {
|
|
format!("Connected to the folder {}.", job.server.root)
|
|
}
|
|
Some(true) => format!("Connected to {name}; the root folder {shown} exists."),
|
|
Some(false) => {
|
|
format!("Connected to {name}, but the root {shown} is a file, not a folder.")
|
|
}
|
|
None => format!(
|
|
"Connected to {name}; the root folder {shown} does not exist yet and is created by the first upload."
|
|
),
|
|
};
|
|
job.monitor.log(Level::Info, message.clone());
|
|
Ok(message)
|
|
}
|
|
|
|
/// The root as shown to the user.
|
|
fn display_root(root: &str) -> &str {
|
|
if root.is_empty() {
|
|
"(login folder)"
|
|
} else {
|
|
root
|
|
}
|
|
}
|
|
|
|
/// Adds the files under the project-relative `rel` to `plan`.
|
|
fn walk_local(
|
|
job: &Job,
|
|
rel: &str,
|
|
depth: usize,
|
|
plan: &mut Vec<Planned>,
|
|
skipped: &mut usize,
|
|
) -> Result<()> {
|
|
job.cancel.check()?;
|
|
let path = if rel.is_empty() {
|
|
job.project_root.clone()
|
|
} else {
|
|
job.project_root.join(rel)
|
|
};
|
|
let meta = std::fs::symlink_metadata(&path).map_err(|e| io_error(&path, &e))?;
|
|
let is_dir = meta.is_dir() || (meta.file_type().is_symlink() && depth == 0 && path.is_dir());
|
|
if !is_dir {
|
|
let size = std::fs::metadata(&path)
|
|
.map_err(|e| io_error(&path, &e))?
|
|
.len();
|
|
match remote_path(&job.server, rel) {
|
|
Some(remote) if !rel.is_empty() => plan.push(Planned {
|
|
local: rel.to_string(),
|
|
remote,
|
|
size,
|
|
}),
|
|
_ => {
|
|
*skipped += 1;
|
|
job.monitor.log(
|
|
Level::Warning,
|
|
format!("{rel}: skipped (excluded or not mapped)"),
|
|
);
|
|
}
|
|
}
|
|
if plan.len() > MAX_FILES {
|
|
return Err(DeployError::Invalid(format!(
|
|
"more than {MAX_FILES} files; upload a smaller folder"
|
|
)));
|
|
}
|
|
return Ok(());
|
|
}
|
|
if depth > MAX_DEPTH || (!rel.is_empty() && is_excluded(&job.server, rel)) {
|
|
return Ok(());
|
|
}
|
|
let mut names: Vec<String> = std::fs::read_dir(&path)
|
|
.map_err(|e| io_error(&path, &e))?
|
|
.filter_map(|e| e.ok())
|
|
.filter(|e| {
|
|
e.file_type()
|
|
.map(|t| !(t.is_symlink() && e.path().is_dir()))
|
|
.unwrap_or(false)
|
|
})
|
|
.map(|e| e.file_name().to_string_lossy().into_owned())
|
|
.filter(|n| !VCS_DIRS.contains(&n.as_str()))
|
|
.collect();
|
|
names.sort();
|
|
for name in names {
|
|
let child = if rel.is_empty() {
|
|
name
|
|
} else {
|
|
format!("{rel}/{name}")
|
|
};
|
|
walk_local(job, &child, depth + 1, plan, skipped)?;
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
/// The parent folder of a remote path; empty for a top-level one.
|
|
fn parent(remote: &str) -> &str {
|
|
match remote.rfind('/') {
|
|
Some(0) => "/",
|
|
Some(i) => remote.get(..i).unwrap_or_default(),
|
|
None => "",
|
|
}
|
|
}
|
|
|
|
/// Whether an error ends the whole job instead of one file.
|
|
fn is_fatal(e: &DeployError) -> bool {
|
|
!matches!(e, DeployError::Transfer(_))
|
|
}
|
|
|
|
/// Uploads the project-relative `paths` (files or folders).
|
|
///
|
|
/// # Errors
|
|
/// [`DeployError::Invalid`] for a path outside the project, otherwise
|
|
/// what [`connect`] returns, or [`DeployError::Cancelled`]. Failing files
|
|
/// are counted in [`Summary::failed`] instead.
|
|
pub fn upload(job: &Job, paths: &[String]) -> Result<Summary> {
|
|
let secret = job.secret.as_ref().map(Secret::expose);
|
|
job.monitor.update(|p| p.phase = "planning".into());
|
|
let mut plan = Vec::new();
|
|
let mut summary = Summary::default();
|
|
for path in paths {
|
|
let rel = normalize_relative(path)
|
|
.ok_or_else(|| DeployError::Invalid(format!("{path:?} is not inside the project")))?;
|
|
walk_local(job, &rel, 0, &mut plan, &mut summary.skipped)?;
|
|
}
|
|
let total: u64 = plan.iter().map(|p| p.size).sum();
|
|
job.monitor.update(|p| {
|
|
p.files_total = plan.len();
|
|
p.bytes_total = total;
|
|
});
|
|
if plan.is_empty() {
|
|
job.monitor.log(Level::Warning, "nothing to upload".into());
|
|
return Ok(summary);
|
|
}
|
|
let mut remote = connect(job)?;
|
|
job.monitor.update(|p| p.phase = "transferring".into());
|
|
let mut made: BTreeSet<String> = BTreeSet::new();
|
|
let result = (|| {
|
|
for file in &plan {
|
|
job.cancel.check()?;
|
|
job.monitor.update(|p| p.file = Some(file.local.clone()));
|
|
let outcome =
|
|
upload_one(job, remote.as_mut(), file, &mut made).map_err(|e| e.redacted(secret));
|
|
job.monitor.update(|p| p.files_done += 1);
|
|
match outcome {
|
|
Ok(()) => {
|
|
summary.files += 1;
|
|
summary.bytes += file.size;
|
|
job.monitor
|
|
.log(Level::Info, format!("{} → {}", file.local, file.remote));
|
|
}
|
|
Err(e) if is_fatal(&e) => return Err(e),
|
|
Err(e) => {
|
|
summary.failed += 1;
|
|
job.monitor
|
|
.log(Level::Error, format!("{}: {e}", file.local));
|
|
}
|
|
}
|
|
}
|
|
Ok(())
|
|
})();
|
|
remote.close();
|
|
result.map(|()| summary)
|
|
}
|
|
|
|
/// Uploads one file, creating its remote folder first.
|
|
fn upload_one(
|
|
job: &Job,
|
|
remote: &mut dyn Remote,
|
|
file: &Planned,
|
|
made: &mut BTreeSet<String>,
|
|
) -> Result<()> {
|
|
let dir = parent(&file.remote).to_string();
|
|
if !dir.is_empty() && dir != "/" && !made.contains(&dir) {
|
|
remote.mkdir_all(&dir)?;
|
|
made.insert(dir);
|
|
}
|
|
let path = job.project_root.join(&file.local);
|
|
let mut data = File::open(&path).map_err(|e| io_error(&path, &e))?;
|
|
let monitor = job.monitor.clone();
|
|
let cancel = job.cancel.clone();
|
|
remote.put(&file.remote, &mut data, &mut |n| {
|
|
monitor.update(|p| p.bytes_done = p.bytes_done.saturating_add(n));
|
|
cancel.check()
|
|
})
|
|
}
|
|
|
|
/// Adds the remote files under the project-relative `rel` to `plan`.
|
|
fn walk_remote(
|
|
job: &Job,
|
|
remote: &mut dyn Remote,
|
|
rel: &str,
|
|
depth: usize,
|
|
plan: &mut Vec<Planned>,
|
|
skipped: &mut usize,
|
|
) -> Result<()> {
|
|
job.cancel.check()?;
|
|
let Some(path) = remote_path(&job.server, rel) else {
|
|
*skipped += 1;
|
|
job.monitor.log(
|
|
Level::Warning,
|
|
format!("{rel}: skipped (excluded or not mapped)"),
|
|
);
|
|
return Ok(());
|
|
};
|
|
match remote.kind(&path)? {
|
|
None => {
|
|
*skipped += 1;
|
|
job.monitor
|
|
.log(Level::Warning, format!("{rel}: not on the server ({path})"));
|
|
}
|
|
Some(false) if !rel.is_empty() => plan.push(Planned {
|
|
local: rel.to_string(),
|
|
remote: path,
|
|
size: 0,
|
|
}),
|
|
Some(false) => {
|
|
*skipped += 1;
|
|
}
|
|
Some(true) => walk_remote_tree(job, remote, rel, &path, depth, plan)?,
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
/// Downloads the project-relative `paths` (files or folders) from the
|
|
/// server into the project, replacing local files.
|
|
///
|
|
/// # Errors
|
|
/// Like [`upload`].
|
|
pub fn download(job: &Job, paths: &[String]) -> Result<Summary> {
|
|
let secret = job.secret.as_ref().map(Secret::expose);
|
|
let mut rels = Vec::new();
|
|
for path in paths {
|
|
rels.push(
|
|
normalize_relative(path).ok_or_else(|| {
|
|
DeployError::Invalid(format!("{path:?} is not inside the project"))
|
|
})?,
|
|
);
|
|
}
|
|
let mut remote = connect(job)?;
|
|
let mut summary = Summary::default();
|
|
let result = (|| {
|
|
job.monitor.update(|p| p.phase = "planning".into());
|
|
let mut plan = Vec::new();
|
|
for rel in &rels {
|
|
walk_remote(
|
|
job,
|
|
remote.as_mut(),
|
|
rel,
|
|
0,
|
|
&mut plan,
|
|
&mut summary.skipped,
|
|
)
|
|
.map_err(|e| e.redacted(secret))?;
|
|
}
|
|
job.monitor.update(|p| {
|
|
p.files_total = plan.len();
|
|
p.phase = "transferring".into();
|
|
});
|
|
for file in &plan {
|
|
job.cancel.check()?;
|
|
job.monitor.update(|p| p.file = Some(file.local.clone()));
|
|
let outcome = download_one(job, remote.as_mut(), file).map_err(|e| e.redacted(secret));
|
|
job.monitor.update(|p| p.files_done += 1);
|
|
match outcome {
|
|
Ok(bytes) => {
|
|
summary.files += 1;
|
|
summary.bytes += bytes;
|
|
job.monitor
|
|
.log(Level::Info, format!("{} ← {}", file.local, file.remote));
|
|
}
|
|
Err(e) if is_fatal(&e) => return Err(e),
|
|
Err(e) => {
|
|
summary.failed += 1;
|
|
job.monitor
|
|
.log(Level::Error, format!("{}: {e}", file.local));
|
|
}
|
|
}
|
|
}
|
|
Ok(())
|
|
})();
|
|
remote.close();
|
|
result.map(|()| summary)
|
|
}
|
|
|
|
/// Downloads one file through a temporary file next to it. Returns its
|
|
/// size. A temporary file that cannot be removed after a failure is left;
|
|
/// the next download of the same file replaces it.
|
|
fn download_one(job: &Job, remote: &mut dyn Remote, file: &Planned) -> Result<u64> {
|
|
let target = job.project_root.join(&file.local);
|
|
if let Some(dir) = target.parent() {
|
|
std::fs::create_dir_all(dir).map_err(|e| io_error(dir, &e))?;
|
|
}
|
|
let temp = target.with_extension("lutra-download");
|
|
let monitor = job.monitor.clone();
|
|
let cancel = job.cancel.clone();
|
|
let mut bytes = 0u64;
|
|
let result = (|| {
|
|
let mut out = File::create(&temp).map_err(|e| io_error(&temp, &e))?;
|
|
remote.get(&file.remote, &mut out, &mut |n| {
|
|
bytes = bytes.saturating_add(n);
|
|
monitor.update(|p| p.bytes_done = p.bytes_done.saturating_add(n));
|
|
cancel.check()
|
|
})?;
|
|
drop(out);
|
|
std::fs::rename(&temp, &target).map_err(|e| io_error(&target, &e))
|
|
})();
|
|
if result.is_err() {
|
|
let _ = std::fs::remove_file(&temp);
|
|
}
|
|
result.map(|()| bytes)
|
|
}
|
|
|
|
/// Adds the files below the remote folder `path` (the mapped folder of the
|
|
/// project-relative `rel`) to `plan`, from its listings: sizes come from
|
|
/// the server, excluded and unmapped entries are left out, and a child
|
|
/// that maps elsewhere is not followed here.
|
|
fn walk_remote_tree(
|
|
job: &Job,
|
|
remote: &mut dyn Remote,
|
|
rel: &str,
|
|
path: &str,
|
|
depth: usize,
|
|
plan: &mut Vec<Planned>,
|
|
) -> Result<()> {
|
|
if depth > MAX_DEPTH {
|
|
return Ok(());
|
|
}
|
|
for entry in remote.list(path)? {
|
|
job.cancel.check()?;
|
|
if VCS_DIRS.contains(&entry.name.as_str()) {
|
|
continue;
|
|
}
|
|
let child = if rel.is_empty() {
|
|
entry.name.clone()
|
|
} else {
|
|
format!("{rel}/{}", entry.name)
|
|
};
|
|
let child_path = join(&[path, &entry.name]);
|
|
if remote_path(&job.server, &child).as_deref() != Some(child_path.as_str()) {
|
|
continue;
|
|
}
|
|
if entry.is_dir {
|
|
walk_remote_tree(job, remote, &child, &child_path, depth + 1, plan)?;
|
|
} else {
|
|
plan.push(Planned {
|
|
local: child,
|
|
remote: child_path,
|
|
size: entry.size.unwrap_or(0),
|
|
});
|
|
}
|
|
if plan.len() > MAX_FILES {
|
|
return Err(DeployError::Invalid(format!(
|
|
"more than {MAX_FILES} files; choose a smaller folder"
|
|
)));
|
|
}
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
/// Largest file whose contents [`compare`] reads to tell equal sizes
|
|
/// apart; larger files of equal size count as equal.
|
|
pub const MAX_COMPARE: u64 = 8 * 1024 * 1024;
|
|
|
|
/// How a file differs between the project and the server.
|
|
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
|
|
#[serde(rename_all = "camelCase")]
|
|
pub enum DiffStatus {
|
|
/// Only in the project.
|
|
LocalOnly,
|
|
/// Only on the server.
|
|
RemoteOnly,
|
|
/// On both, with different contents.
|
|
Different,
|
|
}
|
|
|
|
/// One differing file of [`compare`].
|
|
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
|
|
#[serde(rename_all = "camelCase")]
|
|
pub struct Difference {
|
|
/// Project-relative path.
|
|
pub path: String,
|
|
/// How it differs.
|
|
pub status: DiffStatus,
|
|
/// Its size in the project.
|
|
#[serde(skip_serializing_if = "Option::is_none")]
|
|
pub local_size: Option<u64>,
|
|
/// Its size on the server.
|
|
#[serde(skip_serializing_if = "Option::is_none")]
|
|
pub remote_size: Option<u64>,
|
|
}
|
|
|
|
/// Whether the local `file` and its remote counterpart hold the same
|
|
/// bytes; callers check that both sizes match first.
|
|
fn same_contents(job: &Job, remote: &mut dyn Remote, file: &Planned) -> Result<bool> {
|
|
let path = job.project_root.join(&file.local);
|
|
let local = std::fs::read(&path).map_err(|e| io_error(&path, &e))?;
|
|
let mut theirs = Vec::with_capacity(local.len());
|
|
let cancel = job.cancel.clone();
|
|
remote.get(&file.remote, &mut theirs, &mut |_| cancel.check())?;
|
|
Ok(local == theirs)
|
|
}
|
|
|
|
/// Compares the project-relative `paths` (files or folders) with the
|
|
/// server: files only in the project, only on the server, or with
|
|
/// different contents (sizes first; equal sizes up to [`MAX_COMPARE`] by
|
|
/// contents). Excluded and unmapped files are left out. Sorted by path.
|
|
///
|
|
/// # Errors
|
|
/// [`DeployError::Invalid`] for a path outside the project, otherwise what
|
|
/// the connection or the listing returns.
|
|
pub fn compare(job: &Job, paths: &[String]) -> Result<Vec<Difference>> {
|
|
let secret = job.secret.as_ref().map(Secret::expose);
|
|
job.monitor.update(|p| p.phase = "planning".into());
|
|
let mut local = Vec::new();
|
|
let mut skipped = 0;
|
|
let mut rels = Vec::new();
|
|
for path in paths {
|
|
let rel = normalize_relative(path)
|
|
.ok_or_else(|| DeployError::Invalid(format!("{path:?} is not inside the project")))?;
|
|
if job.project_root.join(&rel).exists() {
|
|
walk_local(job, &rel, 0, &mut local, &mut skipped)?;
|
|
}
|
|
rels.push(rel);
|
|
}
|
|
let mut remote = connect(job)?;
|
|
let result = (|| {
|
|
let mut theirs = Vec::new();
|
|
for rel in &rels {
|
|
let mut ignored = 0;
|
|
walk_remote(job, remote.as_mut(), rel, 0, &mut theirs, &mut ignored)?;
|
|
}
|
|
let mut by_path: BTreeMap<String, Difference> = BTreeMap::new();
|
|
for file in &local {
|
|
by_path.insert(
|
|
file.local.clone(),
|
|
Difference {
|
|
path: file.local.clone(),
|
|
status: DiffStatus::LocalOnly,
|
|
local_size: Some(file.size),
|
|
remote_size: None,
|
|
},
|
|
);
|
|
}
|
|
job.monitor.update(|p| {
|
|
p.phase = "comparing".into();
|
|
p.files_total = theirs.len();
|
|
});
|
|
for file in &theirs {
|
|
job.cancel.check()?;
|
|
job.monitor.update(|p| p.file = Some(file.local.clone()));
|
|
match local.iter().find(|l| l.local == file.local) {
|
|
None => {
|
|
by_path.insert(
|
|
file.local.clone(),
|
|
Difference {
|
|
path: file.local.clone(),
|
|
status: DiffStatus::RemoteOnly,
|
|
local_size: None,
|
|
remote_size: Some(file.size),
|
|
},
|
|
);
|
|
}
|
|
Some(mine) => {
|
|
let differs = mine.size != file.size
|
|
|| (mine.size <= MAX_COMPARE
|
|
&& !same_contents(job, remote.as_mut(), mine)?);
|
|
if differs {
|
|
by_path.insert(
|
|
file.local.clone(),
|
|
Difference {
|
|
path: file.local.clone(),
|
|
status: DiffStatus::Different,
|
|
local_size: Some(mine.size),
|
|
remote_size: Some(file.size),
|
|
},
|
|
);
|
|
} else {
|
|
by_path.remove(&file.local);
|
|
}
|
|
}
|
|
}
|
|
job.monitor.update(|p| p.files_done += 1);
|
|
}
|
|
Ok(by_path.into_values().collect::<Vec<_>>())
|
|
})();
|
|
remote.close();
|
|
let found = result.map_err(|e: DeployError| e.redacted(secret))?;
|
|
job.monitor.log(
|
|
Level::Info,
|
|
format!("compared: {} differing files", found.len()),
|
|
);
|
|
Ok(found)
|
|
}
|
|
|
|
/// Uploads the project-relative files `upload` and deletes the files
|
|
/// `delete` on the server, in one connection. Callers show the plan first
|
|
/// and pass deletions only after the user confirmed them; the engine
|
|
/// refuses deletions without that confirmation.
|
|
///
|
|
/// # Errors
|
|
/// Like [`upload`]; a failing file is counted in [`Summary::failed`].
|
|
pub fn sync(job: &Job, upload: &[String], delete: &[String]) -> Result<Summary> {
|
|
let secret = job.secret.as_ref().map(Secret::expose);
|
|
let mut plan = Vec::new();
|
|
let mut summary = Summary::default();
|
|
for path in upload {
|
|
let rel = normalize_relative(path)
|
|
.ok_or_else(|| DeployError::Invalid(format!("{path:?} is not inside the project")))?;
|
|
walk_local(job, &rel, 0, &mut plan, &mut summary.skipped)?;
|
|
}
|
|
let mut doomed = Vec::new();
|
|
for path in delete {
|
|
let rel = normalize_relative(path)
|
|
.filter(|r| !r.is_empty())
|
|
.ok_or_else(|| DeployError::Invalid(format!("{path:?} is not a project file")))?;
|
|
let remote = remote_path(&job.server, &rel)
|
|
.ok_or_else(|| DeployError::Invalid(format!("{rel} is excluded or not mapped")))?;
|
|
doomed.push((rel, remote));
|
|
}
|
|
job.monitor.update(|p| {
|
|
p.files_total = plan.len() + doomed.len();
|
|
p.bytes_total = plan.iter().map(|f| f.size).sum();
|
|
});
|
|
let mut remote = connect(job)?;
|
|
job.monitor.update(|p| p.phase = "transferring".into());
|
|
let mut made: BTreeSet<String> = BTreeSet::new();
|
|
let result = (|| {
|
|
for file in &plan {
|
|
job.cancel.check()?;
|
|
job.monitor.update(|p| p.file = Some(file.local.clone()));
|
|
let outcome =
|
|
upload_one(job, remote.as_mut(), file, &mut made).map_err(|e| e.redacted(secret));
|
|
job.monitor.update(|p| p.files_done += 1);
|
|
match outcome {
|
|
Ok(()) => {
|
|
summary.files += 1;
|
|
summary.bytes += file.size;
|
|
job.monitor
|
|
.log(Level::Info, format!("{} → {}", file.local, file.remote));
|
|
}
|
|
Err(e) if is_fatal(&e) => return Err(e),
|
|
Err(e) => {
|
|
summary.failed += 1;
|
|
job.monitor
|
|
.log(Level::Error, format!("{}: {e}", file.local));
|
|
}
|
|
}
|
|
}
|
|
for (rel, path) in &doomed {
|
|
job.cancel.check()?;
|
|
job.monitor.update(|p| p.file = Some(rel.clone()));
|
|
let outcome = remote.remove_file(path).map_err(|e| e.redacted(secret));
|
|
job.monitor.update(|p| p.files_done += 1);
|
|
match outcome {
|
|
Ok(()) => {
|
|
summary.deleted += 1;
|
|
job.monitor
|
|
.log(Level::Warning, format!("deleted {path} on the server"));
|
|
}
|
|
Err(e) if is_fatal(&e) => return Err(e),
|
|
Err(e) => {
|
|
summary.failed += 1;
|
|
job.monitor.log(Level::Error, format!("{rel}: {e}"));
|
|
}
|
|
}
|
|
}
|
|
Ok(())
|
|
})();
|
|
remote.close();
|
|
result.map(|()| summary)
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
use crate::config::{AuthKind, Mapping};
|
|
|
|
fn folder_job(project: &Path, target: &Path) -> Job {
|
|
Job {
|
|
server: Server {
|
|
id: "f".into(),
|
|
name: "Mounted".into(),
|
|
kind: ServerKind::Folder,
|
|
host: String::new(),
|
|
port: None,
|
|
user: String::new(),
|
|
root: target.display().to_string(),
|
|
auth: AuthKind::Password,
|
|
key_path: None,
|
|
mappings: vec![Mapping {
|
|
local: String::new(),
|
|
remote: "site".into(),
|
|
}],
|
|
excluded: vec!["*.log".into()],
|
|
upload_on_save: false,
|
|
},
|
|
project_root: project.to_path_buf(),
|
|
secret: None,
|
|
known_hosts: project.join("kh"),
|
|
cancel: Cancel::new(),
|
|
monitor: Monitor::new(),
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn parents() {
|
|
assert_eq!(parent("/a/b"), "/a");
|
|
assert_eq!(parent("/a"), "/");
|
|
assert_eq!(parent("a/b"), "a");
|
|
assert_eq!(parent("a"), "");
|
|
}
|
|
|
|
#[test]
|
|
fn upload_and_download_through_a_folder() {
|
|
let project = tempfile::tempdir().unwrap();
|
|
let target = tempfile::tempdir().unwrap();
|
|
std::fs::create_dir_all(project.path().join("src/sub")).unwrap();
|
|
std::fs::create_dir_all(project.path().join(".git")).unwrap();
|
|
std::fs::write(project.path().join("src/a.php"), "<?php // a").unwrap();
|
|
std::fs::write(project.path().join("src/sub/b.php"), "<?php // b").unwrap();
|
|
std::fs::write(project.path().join("src/x.log"), "log").unwrap();
|
|
std::fs::write(project.path().join(".git/HEAD"), "ref").unwrap();
|
|
let job = folder_job(project.path(), target.path());
|
|
let summary = upload(&job, &[String::new()]).unwrap();
|
|
assert_eq!(summary.files, 2);
|
|
assert_eq!(summary.failed, 0);
|
|
assert_eq!(
|
|
std::fs::read_to_string(target.path().join("site/src/sub/b.php")).unwrap(),
|
|
"<?php // b"
|
|
);
|
|
assert!(!target.path().join("site/src/x.log").exists());
|
|
assert!(!target.path().join("site/.git").exists());
|
|
let progress = job.monitor.progress();
|
|
assert_eq!(progress.files_done, 2);
|
|
assert_eq!(progress.bytes_done, 20);
|
|
assert!(
|
|
job.monitor
|
|
.take_lines()
|
|
.iter()
|
|
.any(|l| l.message.contains("src/a.php → src/a.php")
|
|
|| l.message.contains("src/a.php →"))
|
|
);
|
|
std::fs::write(target.path().join("site/src/a.php"), "<?php // server").unwrap();
|
|
let summary = download(&job, &["src".into()]).unwrap();
|
|
assert_eq!(summary.files, 2);
|
|
assert_eq!(
|
|
std::fs::read_to_string(project.path().join("src/a.php")).unwrap(),
|
|
"<?php // server"
|
|
);
|
|
assert!(upload(&job, &["../x".into()]).is_err());
|
|
let summary = download(&job, &["missing.php".into()]).unwrap();
|
|
assert_eq!(summary.skipped, 1);
|
|
let msg = test_connection(&job).unwrap();
|
|
assert!(msg.contains("Connected"), "{msg}");
|
|
job.cancel.cancel();
|
|
assert_eq!(upload(&job, &["src".into()]), Err(DeployError::Cancelled));
|
|
}
|
|
}
|