D2: remote browser operations, compare and sync in lutra-deploy
Remote trait gains remove_file, remove_dir and rename (SFTP, FTP, folder; test servers too). browse: list, read as text (binary and size caps), write, rename, mkdir, recursive delete that never takes the root. job: compare (local only, server only, different by size then contents) and sync (uploads plus explicitly listed deletions). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
064f40839f
commit
a8ad74755c
10 changed files with 854 additions and 29 deletions
|
|
@ -9,7 +9,7 @@
|
|||
//! Folder walks skip symbolic links to folders and the version-control
|
||||
//! folders `.git`, `.svn` and `.hg`, which are never deployed.
|
||||
|
||||
use std::collections::BTreeSet;
|
||||
use std::collections::{BTreeMap, BTreeSet};
|
||||
use std::fs::File;
|
||||
use std::path::Path;
|
||||
use std::sync::{Arc, Mutex, PoisonError};
|
||||
|
|
@ -18,7 +18,7 @@ use serde::Serialize;
|
|||
|
||||
use crate::config::{Server, ServerKind, io_error};
|
||||
use crate::error::{DeployError, Result};
|
||||
use crate::mapping::{is_excluded, normalize_relative, remote_path};
|
||||
use crate::mapping::{is_excluded, join, normalize_relative, remote_path};
|
||||
use crate::remote::{Cancel, Folder, Remote};
|
||||
use crate::secrets::Secret;
|
||||
|
||||
|
|
@ -56,7 +56,7 @@ pub struct LogLine {
|
|||
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct Progress {
|
||||
/// `connecting`, `planning`, `transferring`.
|
||||
/// `connecting`, `planning`, `comparing`, `transferring`.
|
||||
pub phase: String,
|
||||
/// The file being transferred, project-relative.
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
|
|
@ -117,7 +117,7 @@ impl Monitor {
|
|||
}
|
||||
|
||||
/// Adds a log line.
|
||||
fn log(&self, level: Level, message: String) {
|
||||
pub(crate) fn log(&self, level: Level, message: String) {
|
||||
self.0
|
||||
.lock()
|
||||
.unwrap_or_else(PoisonError::into_inner)
|
||||
|
|
@ -156,6 +156,8 @@ pub struct Summary {
|
|||
pub skipped: usize,
|
||||
/// Files that failed.
|
||||
pub failed: usize,
|
||||
/// Files deleted on the server (sync only).
|
||||
pub deleted: usize,
|
||||
}
|
||||
|
||||
/// One planned file.
|
||||
|
|
@ -427,25 +429,7 @@ fn walk_remote(
|
|||
Some(false) => {
|
||||
*skipped += 1;
|
||||
}
|
||||
Some(true) if depth <= MAX_DEPTH => {
|
||||
for entry in remote.list(&path)? {
|
||||
if VCS_DIRS.contains(&entry.name.as_str()) {
|
||||
continue;
|
||||
}
|
||||
let child = if rel.is_empty() {
|
||||
entry.name
|
||||
} else {
|
||||
format!("{rel}/{}", entry.name)
|
||||
};
|
||||
walk_remote(job, remote, &child, depth + 1, plan, skipped)?;
|
||||
if plan.len() > MAX_FILES {
|
||||
return Err(DeployError::Invalid(format!(
|
||||
"more than {MAX_FILES} files; download a smaller folder"
|
||||
)));
|
||||
}
|
||||
}
|
||||
}
|
||||
Some(true) => {}
|
||||
Some(true) => walk_remote_tree(job, remote, rel, &path, depth, plan)?,
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
|
@ -539,6 +523,266 @@ fn download_one(job: &Job, remote: &mut dyn Remote, file: &Planned) -> Result<u6
|
|||
result.map(|()| bytes)
|
||||
}
|
||||
|
||||
/// Adds the files below the remote folder `path` (the mapped folder of the
|
||||
/// project-relative `rel`) to `plan`, from its listings: sizes come from
|
||||
/// the server, excluded and unmapped entries are left out, and a child
|
||||
/// that maps elsewhere is not followed here.
|
||||
fn walk_remote_tree(
|
||||
job: &Job,
|
||||
remote: &mut dyn Remote,
|
||||
rel: &str,
|
||||
path: &str,
|
||||
depth: usize,
|
||||
plan: &mut Vec<Planned>,
|
||||
) -> Result<()> {
|
||||
if depth > MAX_DEPTH {
|
||||
return Ok(());
|
||||
}
|
||||
for entry in remote.list(path)? {
|
||||
job.cancel.check()?;
|
||||
if VCS_DIRS.contains(&entry.name.as_str()) {
|
||||
continue;
|
||||
}
|
||||
let child = if rel.is_empty() {
|
||||
entry.name.clone()
|
||||
} else {
|
||||
format!("{rel}/{}", entry.name)
|
||||
};
|
||||
let child_path = join(&[path, &entry.name]);
|
||||
if remote_path(&job.server, &child).as_deref() != Some(child_path.as_str()) {
|
||||
continue;
|
||||
}
|
||||
if entry.is_dir {
|
||||
walk_remote_tree(job, remote, &child, &child_path, depth + 1, plan)?;
|
||||
} else {
|
||||
plan.push(Planned {
|
||||
local: child,
|
||||
remote: child_path,
|
||||
size: entry.size.unwrap_or(0),
|
||||
});
|
||||
}
|
||||
if plan.len() > MAX_FILES {
|
||||
return Err(DeployError::Invalid(format!(
|
||||
"more than {MAX_FILES} files; choose a smaller folder"
|
||||
)));
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Largest file whose contents [`compare`] reads to tell equal sizes
|
||||
/// apart; larger files of equal size count as equal.
|
||||
pub const MAX_COMPARE: u64 = 8 * 1024 * 1024;
|
||||
|
||||
/// How a file differs between the project and the server.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub enum DiffStatus {
|
||||
/// Only in the project.
|
||||
LocalOnly,
|
||||
/// Only on the server.
|
||||
RemoteOnly,
|
||||
/// On both, with different contents.
|
||||
Different,
|
||||
}
|
||||
|
||||
/// One differing file of [`compare`].
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct Difference {
|
||||
/// Project-relative path.
|
||||
pub path: String,
|
||||
/// How it differs.
|
||||
pub status: DiffStatus,
|
||||
/// Its size in the project.
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub local_size: Option<u64>,
|
||||
/// Its size on the server.
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub remote_size: Option<u64>,
|
||||
}
|
||||
|
||||
/// Whether the local `file` and its remote counterpart hold the same
|
||||
/// bytes; callers check that both sizes match first.
|
||||
fn same_contents(job: &Job, remote: &mut dyn Remote, file: &Planned) -> Result<bool> {
|
||||
let path = job.project_root.join(&file.local);
|
||||
let local = std::fs::read(&path).map_err(|e| io_error(&path, &e))?;
|
||||
let mut theirs = Vec::with_capacity(local.len());
|
||||
let cancel = job.cancel.clone();
|
||||
remote.get(&file.remote, &mut theirs, &mut |_| cancel.check())?;
|
||||
Ok(local == theirs)
|
||||
}
|
||||
|
||||
/// Compares the project-relative `paths` (files or folders) with the
|
||||
/// server: files only in the project, only on the server, or with
|
||||
/// different contents (sizes first; equal sizes up to [`MAX_COMPARE`] by
|
||||
/// contents). Excluded and unmapped files are left out. Sorted by path.
|
||||
///
|
||||
/// # Errors
|
||||
/// [`DeployError::Invalid`] for a path outside the project, otherwise what
|
||||
/// the connection or the listing returns.
|
||||
pub fn compare(job: &Job, paths: &[String]) -> Result<Vec<Difference>> {
|
||||
let secret = job.secret.as_ref().map(Secret::expose);
|
||||
job.monitor.update(|p| p.phase = "planning".into());
|
||||
let mut local = Vec::new();
|
||||
let mut skipped = 0;
|
||||
let mut rels = Vec::new();
|
||||
for path in paths {
|
||||
let rel = normalize_relative(path)
|
||||
.ok_or_else(|| DeployError::Invalid(format!("{path:?} is not inside the project")))?;
|
||||
if job.project_root.join(&rel).exists() {
|
||||
walk_local(job, &rel, 0, &mut local, &mut skipped)?;
|
||||
}
|
||||
rels.push(rel);
|
||||
}
|
||||
let mut remote = connect(job)?;
|
||||
let result = (|| {
|
||||
let mut theirs = Vec::new();
|
||||
for rel in &rels {
|
||||
let mut ignored = 0;
|
||||
walk_remote(job, remote.as_mut(), rel, 0, &mut theirs, &mut ignored)?;
|
||||
}
|
||||
let mut by_path: BTreeMap<String, Difference> = BTreeMap::new();
|
||||
for file in &local {
|
||||
by_path.insert(
|
||||
file.local.clone(),
|
||||
Difference {
|
||||
path: file.local.clone(),
|
||||
status: DiffStatus::LocalOnly,
|
||||
local_size: Some(file.size),
|
||||
remote_size: None,
|
||||
},
|
||||
);
|
||||
}
|
||||
job.monitor.update(|p| {
|
||||
p.phase = "comparing".into();
|
||||
p.files_total = theirs.len();
|
||||
});
|
||||
for file in &theirs {
|
||||
job.cancel.check()?;
|
||||
job.monitor.update(|p| p.file = Some(file.local.clone()));
|
||||
match local.iter().find(|l| l.local == file.local) {
|
||||
None => {
|
||||
by_path.insert(
|
||||
file.local.clone(),
|
||||
Difference {
|
||||
path: file.local.clone(),
|
||||
status: DiffStatus::RemoteOnly,
|
||||
local_size: None,
|
||||
remote_size: Some(file.size),
|
||||
},
|
||||
);
|
||||
}
|
||||
Some(mine) => {
|
||||
let differs = mine.size != file.size
|
||||
|| (mine.size <= MAX_COMPARE
|
||||
&& !same_contents(job, remote.as_mut(), mine)?);
|
||||
if differs {
|
||||
by_path.insert(
|
||||
file.local.clone(),
|
||||
Difference {
|
||||
path: file.local.clone(),
|
||||
status: DiffStatus::Different,
|
||||
local_size: Some(mine.size),
|
||||
remote_size: Some(file.size),
|
||||
},
|
||||
);
|
||||
} else {
|
||||
by_path.remove(&file.local);
|
||||
}
|
||||
}
|
||||
}
|
||||
job.monitor.update(|p| p.files_done += 1);
|
||||
}
|
||||
Ok(by_path.into_values().collect::<Vec<_>>())
|
||||
})();
|
||||
remote.close();
|
||||
let found = result.map_err(|e: DeployError| e.redacted(secret))?;
|
||||
job.monitor.log(
|
||||
Level::Info,
|
||||
format!("compared: {} differing files", found.len()),
|
||||
);
|
||||
Ok(found)
|
||||
}
|
||||
|
||||
/// Uploads the project-relative files `upload` and deletes the files
|
||||
/// `delete` on the server, in one connection. Callers show the plan first
|
||||
/// and pass deletions only after the user confirmed them; the engine
|
||||
/// refuses deletions without that confirmation.
|
||||
///
|
||||
/// # Errors
|
||||
/// Like [`upload`]; a failing file is counted in [`Summary::failed`].
|
||||
pub fn sync(job: &Job, upload: &[String], delete: &[String]) -> Result<Summary> {
|
||||
let secret = job.secret.as_ref().map(Secret::expose);
|
||||
let mut plan = Vec::new();
|
||||
let mut summary = Summary::default();
|
||||
for path in upload {
|
||||
let rel = normalize_relative(path)
|
||||
.ok_or_else(|| DeployError::Invalid(format!("{path:?} is not inside the project")))?;
|
||||
walk_local(job, &rel, 0, &mut plan, &mut summary.skipped)?;
|
||||
}
|
||||
let mut doomed = Vec::new();
|
||||
for path in delete {
|
||||
let rel = normalize_relative(path)
|
||||
.filter(|r| !r.is_empty())
|
||||
.ok_or_else(|| DeployError::Invalid(format!("{path:?} is not a project file")))?;
|
||||
let remote = remote_path(&job.server, &rel)
|
||||
.ok_or_else(|| DeployError::Invalid(format!("{rel} is excluded or not mapped")))?;
|
||||
doomed.push((rel, remote));
|
||||
}
|
||||
job.monitor.update(|p| {
|
||||
p.files_total = plan.len() + doomed.len();
|
||||
p.bytes_total = plan.iter().map(|f| f.size).sum();
|
||||
});
|
||||
let mut remote = connect(job)?;
|
||||
job.monitor.update(|p| p.phase = "transferring".into());
|
||||
let mut made: BTreeSet<String> = BTreeSet::new();
|
||||
let result = (|| {
|
||||
for file in &plan {
|
||||
job.cancel.check()?;
|
||||
job.monitor.update(|p| p.file = Some(file.local.clone()));
|
||||
let outcome =
|
||||
upload_one(job, remote.as_mut(), file, &mut made).map_err(|e| e.redacted(secret));
|
||||
job.monitor.update(|p| p.files_done += 1);
|
||||
match outcome {
|
||||
Ok(()) => {
|
||||
summary.files += 1;
|
||||
summary.bytes += file.size;
|
||||
job.monitor
|
||||
.log(Level::Info, format!("{} → {}", file.local, file.remote));
|
||||
}
|
||||
Err(e) if is_fatal(&e) => return Err(e),
|
||||
Err(e) => {
|
||||
summary.failed += 1;
|
||||
job.monitor
|
||||
.log(Level::Error, format!("{}: {e}", file.local));
|
||||
}
|
||||
}
|
||||
}
|
||||
for (rel, path) in &doomed {
|
||||
job.cancel.check()?;
|
||||
job.monitor.update(|p| p.file = Some(rel.clone()));
|
||||
let outcome = remote.remove_file(path).map_err(|e| e.redacted(secret));
|
||||
job.monitor.update(|p| p.files_done += 1);
|
||||
match outcome {
|
||||
Ok(()) => {
|
||||
summary.deleted += 1;
|
||||
job.monitor
|
||||
.log(Level::Warning, format!("deleted {path} on the server"));
|
||||
}
|
||||
Err(e) if is_fatal(&e) => return Err(e),
|
||||
Err(e) => {
|
||||
summary.failed += 1;
|
||||
job.monitor.log(Level::Error, format!("{rel}: {e}"));
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
})();
|
||||
remote.close();
|
||||
result.map(|()| summary)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue