D2: remote browser operations, compare and sync in lutra-deploy

Remote trait gains remove_file, remove_dir and rename (SFTP, FTP,
folder; test servers too). browse: list, read as text (binary and size
caps), write, rename, mkdir, recursive delete that never takes the root.
job: compare (local only, server only, different by size then
contents) and sync (uploads plus explicitly listed deletions).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
danny 2026-11-06 19:38:07 +01:00
parent 064f40839f
commit a8ad74755c
10 changed files with 854 additions and 29 deletions

View file

@ -9,7 +9,7 @@
//! Folder walks skip symbolic links to folders and the version-control
//! folders `.git`, `.svn` and `.hg`, which are never deployed.
use std::collections::BTreeSet;
use std::collections::{BTreeMap, BTreeSet};
use std::fs::File;
use std::path::Path;
use std::sync::{Arc, Mutex, PoisonError};
@ -18,7 +18,7 @@ use serde::Serialize;
use crate::config::{Server, ServerKind, io_error};
use crate::error::{DeployError, Result};
use crate::mapping::{is_excluded, normalize_relative, remote_path};
use crate::mapping::{is_excluded, join, normalize_relative, remote_path};
use crate::remote::{Cancel, Folder, Remote};
use crate::secrets::Secret;
@ -56,7 +56,7 @@ pub struct LogLine {
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct Progress {
/// `connecting`, `planning`, `transferring`.
/// `connecting`, `planning`, `comparing`, `transferring`.
pub phase: String,
/// The file being transferred, project-relative.
#[serde(skip_serializing_if = "Option::is_none")]
@ -117,7 +117,7 @@ impl Monitor {
}
/// Adds a log line.
fn log(&self, level: Level, message: String) {
pub(crate) fn log(&self, level: Level, message: String) {
self.0
.lock()
.unwrap_or_else(PoisonError::into_inner)
@ -156,6 +156,8 @@ pub struct Summary {
pub skipped: usize,
/// Files that failed.
pub failed: usize,
/// Files deleted on the server (sync only).
pub deleted: usize,
}
/// One planned file.
@ -427,25 +429,7 @@ fn walk_remote(
Some(false) => {
*skipped += 1;
}
Some(true) if depth <= MAX_DEPTH => {
for entry in remote.list(&path)? {
if VCS_DIRS.contains(&entry.name.as_str()) {
continue;
}
let child = if rel.is_empty() {
entry.name
} else {
format!("{rel}/{}", entry.name)
};
walk_remote(job, remote, &child, depth + 1, plan, skipped)?;
if plan.len() > MAX_FILES {
return Err(DeployError::Invalid(format!(
"more than {MAX_FILES} files; download a smaller folder"
)));
}
}
}
Some(true) => {}
Some(true) => walk_remote_tree(job, remote, rel, &path, depth, plan)?,
}
Ok(())
}
@ -539,6 +523,266 @@ fn download_one(job: &Job, remote: &mut dyn Remote, file: &Planned) -> Result<u6
result.map(|()| bytes)
}
/// Adds the files below the remote folder `path` (the mapped folder of the
/// project-relative `rel`) to `plan`, from its listings: sizes come from
/// the server, excluded and unmapped entries are left out, and a child
/// that maps elsewhere is not followed here.
fn walk_remote_tree(
job: &Job,
remote: &mut dyn Remote,
rel: &str,
path: &str,
depth: usize,
plan: &mut Vec<Planned>,
) -> Result<()> {
if depth > MAX_DEPTH {
return Ok(());
}
for entry in remote.list(path)? {
job.cancel.check()?;
if VCS_DIRS.contains(&entry.name.as_str()) {
continue;
}
let child = if rel.is_empty() {
entry.name.clone()
} else {
format!("{rel}/{}", entry.name)
};
let child_path = join(&[path, &entry.name]);
if remote_path(&job.server, &child).as_deref() != Some(child_path.as_str()) {
continue;
}
if entry.is_dir {
walk_remote_tree(job, remote, &child, &child_path, depth + 1, plan)?;
} else {
plan.push(Planned {
local: child,
remote: child_path,
size: entry.size.unwrap_or(0),
});
}
if plan.len() > MAX_FILES {
return Err(DeployError::Invalid(format!(
"more than {MAX_FILES} files; choose a smaller folder"
)));
}
}
Ok(())
}
/// Largest file whose contents [`compare`] reads to tell equal sizes
/// apart; larger files of equal size count as equal.
pub const MAX_COMPARE: u64 = 8 * 1024 * 1024;
/// How a file differs between the project and the server.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
#[serde(rename_all = "camelCase")]
pub enum DiffStatus {
/// Only in the project.
LocalOnly,
/// Only on the server.
RemoteOnly,
/// On both, with different contents.
Different,
}
/// One differing file of [`compare`].
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct Difference {
/// Project-relative path.
pub path: String,
/// How it differs.
pub status: DiffStatus,
/// Its size in the project.
#[serde(skip_serializing_if = "Option::is_none")]
pub local_size: Option<u64>,
/// Its size on the server.
#[serde(skip_serializing_if = "Option::is_none")]
pub remote_size: Option<u64>,
}
/// Whether the local `file` and its remote counterpart hold the same
/// bytes; callers check that both sizes match first.
fn same_contents(job: &Job, remote: &mut dyn Remote, file: &Planned) -> Result<bool> {
let path = job.project_root.join(&file.local);
let local = std::fs::read(&path).map_err(|e| io_error(&path, &e))?;
let mut theirs = Vec::with_capacity(local.len());
let cancel = job.cancel.clone();
remote.get(&file.remote, &mut theirs, &mut |_| cancel.check())?;
Ok(local == theirs)
}
/// Compares the project-relative `paths` (files or folders) with the
/// server: files only in the project, only on the server, or with
/// different contents (sizes first; equal sizes up to [`MAX_COMPARE`] by
/// contents). Excluded and unmapped files are left out. Sorted by path.
///
/// # Errors
/// [`DeployError::Invalid`] for a path outside the project, otherwise what
/// the connection or the listing returns.
pub fn compare(job: &Job, paths: &[String]) -> Result<Vec<Difference>> {
let secret = job.secret.as_ref().map(Secret::expose);
job.monitor.update(|p| p.phase = "planning".into());
let mut local = Vec::new();
let mut skipped = 0;
let mut rels = Vec::new();
for path in paths {
let rel = normalize_relative(path)
.ok_or_else(|| DeployError::Invalid(format!("{path:?} is not inside the project")))?;
if job.project_root.join(&rel).exists() {
walk_local(job, &rel, 0, &mut local, &mut skipped)?;
}
rels.push(rel);
}
let mut remote = connect(job)?;
let result = (|| {
let mut theirs = Vec::new();
for rel in &rels {
let mut ignored = 0;
walk_remote(job, remote.as_mut(), rel, 0, &mut theirs, &mut ignored)?;
}
let mut by_path: BTreeMap<String, Difference> = BTreeMap::new();
for file in &local {
by_path.insert(
file.local.clone(),
Difference {
path: file.local.clone(),
status: DiffStatus::LocalOnly,
local_size: Some(file.size),
remote_size: None,
},
);
}
job.monitor.update(|p| {
p.phase = "comparing".into();
p.files_total = theirs.len();
});
for file in &theirs {
job.cancel.check()?;
job.monitor.update(|p| p.file = Some(file.local.clone()));
match local.iter().find(|l| l.local == file.local) {
None => {
by_path.insert(
file.local.clone(),
Difference {
path: file.local.clone(),
status: DiffStatus::RemoteOnly,
local_size: None,
remote_size: Some(file.size),
},
);
}
Some(mine) => {
let differs = mine.size != file.size
|| (mine.size <= MAX_COMPARE
&& !same_contents(job, remote.as_mut(), mine)?);
if differs {
by_path.insert(
file.local.clone(),
Difference {
path: file.local.clone(),
status: DiffStatus::Different,
local_size: Some(mine.size),
remote_size: Some(file.size),
},
);
} else {
by_path.remove(&file.local);
}
}
}
job.monitor.update(|p| p.files_done += 1);
}
Ok(by_path.into_values().collect::<Vec<_>>())
})();
remote.close();
let found = result.map_err(|e: DeployError| e.redacted(secret))?;
job.monitor.log(
Level::Info,
format!("compared: {} differing files", found.len()),
);
Ok(found)
}
/// Uploads the project-relative files `upload` and deletes the files
/// `delete` on the server, in one connection. Callers show the plan first
/// and pass deletions only after the user confirmed them; the engine
/// refuses deletions without that confirmation.
///
/// # Errors
/// Like [`upload`]; a failing file is counted in [`Summary::failed`].
pub fn sync(job: &Job, upload: &[String], delete: &[String]) -> Result<Summary> {
let secret = job.secret.as_ref().map(Secret::expose);
let mut plan = Vec::new();
let mut summary = Summary::default();
for path in upload {
let rel = normalize_relative(path)
.ok_or_else(|| DeployError::Invalid(format!("{path:?} is not inside the project")))?;
walk_local(job, &rel, 0, &mut plan, &mut summary.skipped)?;
}
let mut doomed = Vec::new();
for path in delete {
let rel = normalize_relative(path)
.filter(|r| !r.is_empty())
.ok_or_else(|| DeployError::Invalid(format!("{path:?} is not a project file")))?;
let remote = remote_path(&job.server, &rel)
.ok_or_else(|| DeployError::Invalid(format!("{rel} is excluded or not mapped")))?;
doomed.push((rel, remote));
}
job.monitor.update(|p| {
p.files_total = plan.len() + doomed.len();
p.bytes_total = plan.iter().map(|f| f.size).sum();
});
let mut remote = connect(job)?;
job.monitor.update(|p| p.phase = "transferring".into());
let mut made: BTreeSet<String> = BTreeSet::new();
let result = (|| {
for file in &plan {
job.cancel.check()?;
job.monitor.update(|p| p.file = Some(file.local.clone()));
let outcome =
upload_one(job, remote.as_mut(), file, &mut made).map_err(|e| e.redacted(secret));
job.monitor.update(|p| p.files_done += 1);
match outcome {
Ok(()) => {
summary.files += 1;
summary.bytes += file.size;
job.monitor
.log(Level::Info, format!("{} → {}", file.local, file.remote));
}
Err(e) if is_fatal(&e) => return Err(e),
Err(e) => {
summary.failed += 1;
job.monitor
.log(Level::Error, format!("{}: {e}", file.local));
}
}
}
for (rel, path) in &doomed {
job.cancel.check()?;
job.monitor.update(|p| p.file = Some(rel.clone()));
let outcome = remote.remove_file(path).map_err(|e| e.redacted(secret));
job.monitor.update(|p| p.files_done += 1);
match outcome {
Ok(()) => {
summary.deleted += 1;
job.monitor
.log(Level::Warning, format!("deleted {path} on the server"));
}
Err(e) if is_fatal(&e) => return Err(e),
Err(e) => {
summary.failed += 1;
job.monitor.log(Level::Error, format!("{rel}: {e}"));
}
}
}
Ok(())
})();
remote.close();
result.map(|()| summary)
}
#[cfg(test)]
mod tests {
use super::*;