D2: remote browser operations, compare and sync in lutra-deploy
Remote trait gains remove_file, remove_dir and rename (SFTP, FTP, folder; test servers too). browse: list, read as text (binary and size caps), write, rename, mkdir, recursive delete that never takes the root. job: compare (local only, server only, different by size then contents) and sync (uploads plus explicitly listed deletions). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
064f40839f
commit
a8ad74755c
10 changed files with 854 additions and 29 deletions
|
|
@ -41,3 +41,7 @@ required-features = ["testserver"]
|
|||
[[example]]
|
||||
name = "testserver"
|
||||
required-features = ["testserver"]
|
||||
|
||||
[[test]]
|
||||
name = "remote"
|
||||
required-features = ["testserver"]
|
||||
|
|
|
|||
296
crates/deploy/src/browse.rs
Normal file
296
crates/deploy/src/browse.rs
Normal file
|
|
@ -0,0 +1,296 @@
|
|||
//! The remote browser's operations (D2): list a folder, read and write a
|
||||
//! file, rename, delete and create folders on the server.
|
||||
//!
|
||||
//! Browser paths are relative to the server's root with `/` (`""` is the
|
||||
//! root itself); `..`, control characters and absolute paths are refused,
|
||||
//! so the browser never leaves the deployment root. Every operation opens
|
||||
//! its own connection and closes it.
|
||||
|
||||
use crate::ServerKind;
|
||||
use crate::error::{DeployError, Result};
|
||||
use crate::job::{Job, Level, connect};
|
||||
use crate::mapping::{join, normalize_relative, remote_path};
|
||||
use crate::remote::{Entry, Remote};
|
||||
use crate::secrets::Secret;
|
||||
|
||||
/// Largest file [`read`] returns as text.
|
||||
pub const MAX_TEXT: u64 = 4 * 1024 * 1024;
|
||||
|
||||
/// What [`read`] found.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Default)]
|
||||
pub struct RemoteText {
|
||||
/// The UTF-8 text; `None` when the file does not exist on the server.
|
||||
pub text: Option<String>,
|
||||
/// The file is not UTF-8 text (or holds a NUL byte).
|
||||
pub binary: bool,
|
||||
/// The file is larger than [`MAX_TEXT`].
|
||||
pub too_large: bool,
|
||||
}
|
||||
|
||||
/// Which file [`read`] reads.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub enum Target {
|
||||
/// A project-relative path, mapped through the server's mappings.
|
||||
Mapped(String),
|
||||
/// A browser path, relative to the server's root.
|
||||
Browser(String),
|
||||
}
|
||||
|
||||
/// The server path of the browser path `rel`.
|
||||
///
|
||||
/// # Errors
|
||||
/// [`DeployError::Invalid`] for an absolute path, `..` or control
|
||||
/// characters.
|
||||
pub fn server_path(job: &Job, rel: &str) -> Result<String> {
|
||||
if rel.starts_with('/') || rel.starts_with('\\') {
|
||||
return Err(DeployError::Invalid(format!(
|
||||
"{rel:?} must be relative to the server's root folder"
|
||||
)));
|
||||
}
|
||||
let rel = normalize_relative(rel)
|
||||
.ok_or_else(|| DeployError::Invalid(format!("{rel:?} leaves the server's root folder")))?;
|
||||
Ok(if job.server.kind == ServerKind::Folder {
|
||||
rel
|
||||
} else {
|
||||
join(&[&job.server.root, &rel])
|
||||
})
|
||||
}
|
||||
|
||||
/// Runs `op` on a fresh connection and closes it; errors are redacted.
|
||||
fn with_remote<T>(job: &Job, op: impl FnOnce(&mut dyn Remote) -> Result<T>) -> Result<T> {
|
||||
let secret = job.secret.as_ref().map(Secret::expose);
|
||||
let mut remote = connect(job)?;
|
||||
let result = op(remote.as_mut());
|
||||
remote.close();
|
||||
result.map_err(|e| e.redacted(secret))
|
||||
}
|
||||
|
||||
/// The entries of the browser folder `rel`, folders first, then by name.
|
||||
///
|
||||
/// # Errors
|
||||
/// What the connection or the listing returns.
|
||||
pub fn list(job: &Job, rel: &str) -> Result<Vec<Entry>> {
|
||||
let path = server_path(job, rel)?;
|
||||
with_remote(job, |remote| {
|
||||
let mut entries = remote.list(&path)?;
|
||||
entries.sort_by(|a, b| b.is_dir.cmp(&a.is_dir).then_with(|| a.name.cmp(&b.name)));
|
||||
Ok(entries)
|
||||
})
|
||||
}
|
||||
|
||||
/// Reads one file as text, at most [`MAX_TEXT`] bytes.
|
||||
///
|
||||
/// # Errors
|
||||
/// [`DeployError::Invalid`] for a path that is excluded or not mapped,
|
||||
/// otherwise what the connection or the transfer returns.
|
||||
pub fn read(job: &Job, target: &Target) -> Result<RemoteText> {
|
||||
let path = match target {
|
||||
Target::Mapped(rel) => remote_path(&job.server, rel).ok_or_else(|| {
|
||||
DeployError::Invalid(format!(
|
||||
"{rel} is excluded or not mapped on {}",
|
||||
job.server.name
|
||||
))
|
||||
})?,
|
||||
Target::Browser(rel) => server_path(job, rel)?,
|
||||
};
|
||||
with_remote(job, |remote| {
|
||||
match remote.kind(&path)? {
|
||||
None => return Ok(RemoteText::default()),
|
||||
Some(true) => {
|
||||
return Err(DeployError::Invalid(format!(
|
||||
"{path} is a folder on the server"
|
||||
)));
|
||||
}
|
||||
Some(false) => {}
|
||||
}
|
||||
let mut bytes = Vec::new();
|
||||
let mut total = 0u64;
|
||||
let cancel = job.cancel.clone();
|
||||
let read = remote.get(&path, &mut bytes, &mut |n| {
|
||||
total = total.saturating_add(n);
|
||||
if total > MAX_TEXT {
|
||||
return Err(DeployError::Cancelled);
|
||||
}
|
||||
cancel.check()
|
||||
});
|
||||
match read {
|
||||
Err(DeployError::Cancelled) if total > MAX_TEXT => Ok(RemoteText {
|
||||
text: None,
|
||||
binary: false,
|
||||
too_large: true,
|
||||
}),
|
||||
Err(e) => Err(e),
|
||||
Ok(()) => Ok(match String::from_utf8(bytes) {
|
||||
Ok(text) if !text.contains('\0') => RemoteText {
|
||||
text: Some(text),
|
||||
binary: false,
|
||||
too_large: false,
|
||||
},
|
||||
_ => RemoteText {
|
||||
text: None,
|
||||
binary: true,
|
||||
too_large: false,
|
||||
},
|
||||
}),
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
/// Writes `bytes` to the browser path `rel`, creating its folder.
|
||||
///
|
||||
/// # Errors
|
||||
/// What the connection or the transfer returns.
|
||||
pub fn write(job: &Job, rel: &str, bytes: &[u8]) -> Result<()> {
|
||||
let path = server_path(job, rel)?;
|
||||
let cancel = job.cancel.clone();
|
||||
with_remote(job, |remote| {
|
||||
if let Some(slash) = path.rfind('/')
|
||||
&& slash > 0
|
||||
{
|
||||
remote.mkdir_all(path.get(..slash).unwrap_or_default())?;
|
||||
}
|
||||
remote.put(&path, &mut &bytes[..], &mut |_| cancel.check())
|
||||
})?;
|
||||
job.monitor
|
||||
.log(Level::Info, format!("saved {path} on the server"));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Renames or moves the browser path `from` to `to`.
|
||||
///
|
||||
/// # Errors
|
||||
/// What the connection or the server returns.
|
||||
pub fn rename(job: &Job, from: &str, to: &str) -> Result<()> {
|
||||
let from = server_path(job, from)?;
|
||||
let to = server_path(job, to)?;
|
||||
with_remote(job, |remote| remote.rename(&from, &to))?;
|
||||
job.monitor
|
||||
.log(Level::Info, format!("renamed {from} to {to}"));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Creates the browser folder `rel` and its missing parents.
|
||||
///
|
||||
/// # Errors
|
||||
/// What the connection or the server returns.
|
||||
pub fn mkdir(job: &Job, rel: &str) -> Result<()> {
|
||||
let path = server_path(job, rel)?;
|
||||
with_remote(job, |remote| remote.mkdir_all(&path))?;
|
||||
job.monitor
|
||||
.log(Level::Info, format!("created the folder {path}"));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Removes `path` on the server; a folder with everything inside it.
|
||||
/// Returns how many files and folders were removed.
|
||||
fn remove_tree(job: &Job, remote: &mut dyn Remote, path: &str, depth: usize) -> Result<usize> {
|
||||
job.cancel.check()?;
|
||||
match remote.kind(path)? {
|
||||
None => Ok(0),
|
||||
Some(false) => {
|
||||
remote.remove_file(path)?;
|
||||
job.monitor.log(Level::Info, format!("deleted {path}"));
|
||||
Ok(1)
|
||||
}
|
||||
Some(true) => {
|
||||
if depth > 64 {
|
||||
return Err(DeployError::Transfer(format!(
|
||||
"{path} is nested too deeply"
|
||||
)));
|
||||
}
|
||||
let mut count = 0;
|
||||
for entry in remote.list(path)? {
|
||||
count += remove_tree(job, remote, &format!("{path}/{}", entry.name), depth + 1)?;
|
||||
}
|
||||
remote.remove_dir(path)?;
|
||||
job.monitor
|
||||
.log(Level::Info, format!("deleted the folder {path}"));
|
||||
Ok(count + 1)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Deletes the browser path `rel` on the server, a folder with its
|
||||
/// contents. The root itself cannot be deleted. Callers ask the user
|
||||
/// first; the engine refuses without an explicit confirmation.
|
||||
///
|
||||
/// # Errors
|
||||
/// [`DeployError::Invalid`] for the root, otherwise what the connection or
|
||||
/// the server returns.
|
||||
pub fn delete(job: &Job, rel: &str) -> Result<usize> {
|
||||
let path = server_path(job, rel)?;
|
||||
if normalize_relative(rel).unwrap_or_default().is_empty() {
|
||||
return Err(DeployError::Invalid(
|
||||
"the server's root folder cannot be deleted from Lutra".into(),
|
||||
));
|
||||
}
|
||||
with_remote(job, |remote| remove_tree(job, remote, &path, 0))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::config::{AuthKind, Server};
|
||||
use crate::{Cancel, Monitor};
|
||||
|
||||
fn job(root: &std::path::Path) -> Job {
|
||||
Job {
|
||||
server: Server {
|
||||
id: "f".into(),
|
||||
name: "F".into(),
|
||||
kind: ServerKind::Folder,
|
||||
host: String::new(),
|
||||
port: None,
|
||||
user: String::new(),
|
||||
root: root.display().to_string(),
|
||||
auth: AuthKind::Password,
|
||||
key_path: None,
|
||||
mappings: vec![],
|
||||
excluded: vec![],
|
||||
upload_on_save: false,
|
||||
},
|
||||
project_root: root.to_path_buf(),
|
||||
secret: None,
|
||||
known_hosts: root.join("kh"),
|
||||
cancel: Cancel::new(),
|
||||
monitor: Monitor::new(),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn browser_operations_on_a_folder() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let j = job(dir.path());
|
||||
mkdir(&j, "a/b").unwrap();
|
||||
write(&j, "a/b/x.php", b"<?php").unwrap();
|
||||
write(&j, "a/bin.dat", &[0, 1, 2]).unwrap();
|
||||
let entries = list(&j, "a").unwrap();
|
||||
assert_eq!(
|
||||
entries.iter().map(|e| e.name.as_str()).collect::<Vec<_>>(),
|
||||
["b", "bin.dat"]
|
||||
);
|
||||
assert_eq!(
|
||||
read(&j, &Target::Browser("a/b/x.php".into()))
|
||||
.unwrap()
|
||||
.text
|
||||
.as_deref(),
|
||||
Some("<?php")
|
||||
);
|
||||
assert!(
|
||||
read(&j, &Target::Browser("a/bin.dat".into()))
|
||||
.unwrap()
|
||||
.binary
|
||||
);
|
||||
assert_eq!(
|
||||
read(&j, &Target::Browser("nope".into())).unwrap(),
|
||||
RemoteText::default()
|
||||
);
|
||||
rename(&j, "a/b/x.php", "a/y.php").unwrap();
|
||||
assert!(dir.path().join("a/y.php").is_file());
|
||||
assert_eq!(delete(&j, "a").unwrap(), 4);
|
||||
assert!(!dir.path().join("a").exists());
|
||||
assert!(delete(&j, "").is_err());
|
||||
assert!(list(&j, "../up").is_err());
|
||||
assert!(list(&j, "/etc").is_err());
|
||||
}
|
||||
}
|
||||
|
|
@ -240,6 +240,24 @@ impl Remote for Ftp {
|
|||
Ok(entries)
|
||||
}
|
||||
|
||||
fn remove_file(&mut self, path: &str) -> Result<()> {
|
||||
self.ftp()
|
||||
.rm(path)
|
||||
.map_err(|e| ftp_error(&format!("cannot remove {path}"), &e))
|
||||
}
|
||||
|
||||
fn remove_dir(&mut self, path: &str) -> Result<()> {
|
||||
self.ftp()
|
||||
.rmdir(path)
|
||||
.map_err(|e| ftp_error(&format!("cannot remove the folder {path}"), &e))
|
||||
}
|
||||
|
||||
fn rename(&mut self, from: &str, to: &str) -> Result<()> {
|
||||
self.ftp()
|
||||
.rename(from, to)
|
||||
.map_err(|e| ftp_error(&format!("cannot rename {from} to {to}"), &e))
|
||||
}
|
||||
|
||||
/// Sends `QUIT`; a failure is ignored because the connection is
|
||||
/// dropped right after.
|
||||
fn close(&mut self) {
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@
|
|||
//! Folder walks skip symbolic links to folders and the version-control
|
||||
//! folders `.git`, `.svn` and `.hg`, which are never deployed.
|
||||
|
||||
use std::collections::BTreeSet;
|
||||
use std::collections::{BTreeMap, BTreeSet};
|
||||
use std::fs::File;
|
||||
use std::path::Path;
|
||||
use std::sync::{Arc, Mutex, PoisonError};
|
||||
|
|
@ -18,7 +18,7 @@ use serde::Serialize;
|
|||
|
||||
use crate::config::{Server, ServerKind, io_error};
|
||||
use crate::error::{DeployError, Result};
|
||||
use crate::mapping::{is_excluded, normalize_relative, remote_path};
|
||||
use crate::mapping::{is_excluded, join, normalize_relative, remote_path};
|
||||
use crate::remote::{Cancel, Folder, Remote};
|
||||
use crate::secrets::Secret;
|
||||
|
||||
|
|
@ -56,7 +56,7 @@ pub struct LogLine {
|
|||
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct Progress {
|
||||
/// `connecting`, `planning`, `transferring`.
|
||||
/// `connecting`, `planning`, `comparing`, `transferring`.
|
||||
pub phase: String,
|
||||
/// The file being transferred, project-relative.
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
|
|
@ -117,7 +117,7 @@ impl Monitor {
|
|||
}
|
||||
|
||||
/// Adds a log line.
|
||||
fn log(&self, level: Level, message: String) {
|
||||
pub(crate) fn log(&self, level: Level, message: String) {
|
||||
self.0
|
||||
.lock()
|
||||
.unwrap_or_else(PoisonError::into_inner)
|
||||
|
|
@ -156,6 +156,8 @@ pub struct Summary {
|
|||
pub skipped: usize,
|
||||
/// Files that failed.
|
||||
pub failed: usize,
|
||||
/// Files deleted on the server (sync only).
|
||||
pub deleted: usize,
|
||||
}
|
||||
|
||||
/// One planned file.
|
||||
|
|
@ -427,25 +429,7 @@ fn walk_remote(
|
|||
Some(false) => {
|
||||
*skipped += 1;
|
||||
}
|
||||
Some(true) if depth <= MAX_DEPTH => {
|
||||
for entry in remote.list(&path)? {
|
||||
if VCS_DIRS.contains(&entry.name.as_str()) {
|
||||
continue;
|
||||
}
|
||||
let child = if rel.is_empty() {
|
||||
entry.name
|
||||
} else {
|
||||
format!("{rel}/{}", entry.name)
|
||||
};
|
||||
walk_remote(job, remote, &child, depth + 1, plan, skipped)?;
|
||||
if plan.len() > MAX_FILES {
|
||||
return Err(DeployError::Invalid(format!(
|
||||
"more than {MAX_FILES} files; download a smaller folder"
|
||||
)));
|
||||
}
|
||||
}
|
||||
}
|
||||
Some(true) => {}
|
||||
Some(true) => walk_remote_tree(job, remote, rel, &path, depth, plan)?,
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
|
@ -539,6 +523,266 @@ fn download_one(job: &Job, remote: &mut dyn Remote, file: &Planned) -> Result<u6
|
|||
result.map(|()| bytes)
|
||||
}
|
||||
|
||||
/// Adds the files below the remote folder `path` (the mapped folder of the
|
||||
/// project-relative `rel`) to `plan`, from its listings: sizes come from
|
||||
/// the server, excluded and unmapped entries are left out, and a child
|
||||
/// that maps elsewhere is not followed here.
|
||||
fn walk_remote_tree(
|
||||
job: &Job,
|
||||
remote: &mut dyn Remote,
|
||||
rel: &str,
|
||||
path: &str,
|
||||
depth: usize,
|
||||
plan: &mut Vec<Planned>,
|
||||
) -> Result<()> {
|
||||
if depth > MAX_DEPTH {
|
||||
return Ok(());
|
||||
}
|
||||
for entry in remote.list(path)? {
|
||||
job.cancel.check()?;
|
||||
if VCS_DIRS.contains(&entry.name.as_str()) {
|
||||
continue;
|
||||
}
|
||||
let child = if rel.is_empty() {
|
||||
entry.name.clone()
|
||||
} else {
|
||||
format!("{rel}/{}", entry.name)
|
||||
};
|
||||
let child_path = join(&[path, &entry.name]);
|
||||
if remote_path(&job.server, &child).as_deref() != Some(child_path.as_str()) {
|
||||
continue;
|
||||
}
|
||||
if entry.is_dir {
|
||||
walk_remote_tree(job, remote, &child, &child_path, depth + 1, plan)?;
|
||||
} else {
|
||||
plan.push(Planned {
|
||||
local: child,
|
||||
remote: child_path,
|
||||
size: entry.size.unwrap_or(0),
|
||||
});
|
||||
}
|
||||
if plan.len() > MAX_FILES {
|
||||
return Err(DeployError::Invalid(format!(
|
||||
"more than {MAX_FILES} files; choose a smaller folder"
|
||||
)));
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Largest file whose contents [`compare`] reads to tell equal sizes
|
||||
/// apart; larger files of equal size count as equal.
|
||||
pub const MAX_COMPARE: u64 = 8 * 1024 * 1024;
|
||||
|
||||
/// How a file differs between the project and the server.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub enum DiffStatus {
|
||||
/// Only in the project.
|
||||
LocalOnly,
|
||||
/// Only on the server.
|
||||
RemoteOnly,
|
||||
/// On both, with different contents.
|
||||
Different,
|
||||
}
|
||||
|
||||
/// One differing file of [`compare`].
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct Difference {
|
||||
/// Project-relative path.
|
||||
pub path: String,
|
||||
/// How it differs.
|
||||
pub status: DiffStatus,
|
||||
/// Its size in the project.
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub local_size: Option<u64>,
|
||||
/// Its size on the server.
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub remote_size: Option<u64>,
|
||||
}
|
||||
|
||||
/// Whether the local `file` and its remote counterpart hold the same
|
||||
/// bytes; callers check that both sizes match first.
|
||||
fn same_contents(job: &Job, remote: &mut dyn Remote, file: &Planned) -> Result<bool> {
|
||||
let path = job.project_root.join(&file.local);
|
||||
let local = std::fs::read(&path).map_err(|e| io_error(&path, &e))?;
|
||||
let mut theirs = Vec::with_capacity(local.len());
|
||||
let cancel = job.cancel.clone();
|
||||
remote.get(&file.remote, &mut theirs, &mut |_| cancel.check())?;
|
||||
Ok(local == theirs)
|
||||
}
|
||||
|
||||
/// Compares the project-relative `paths` (files or folders) with the
|
||||
/// server: files only in the project, only on the server, or with
|
||||
/// different contents (sizes first; equal sizes up to [`MAX_COMPARE`] by
|
||||
/// contents). Excluded and unmapped files are left out. Sorted by path.
|
||||
///
|
||||
/// # Errors
|
||||
/// [`DeployError::Invalid`] for a path outside the project, otherwise what
|
||||
/// the connection or the listing returns.
|
||||
pub fn compare(job: &Job, paths: &[String]) -> Result<Vec<Difference>> {
|
||||
let secret = job.secret.as_ref().map(Secret::expose);
|
||||
job.monitor.update(|p| p.phase = "planning".into());
|
||||
let mut local = Vec::new();
|
||||
let mut skipped = 0;
|
||||
let mut rels = Vec::new();
|
||||
for path in paths {
|
||||
let rel = normalize_relative(path)
|
||||
.ok_or_else(|| DeployError::Invalid(format!("{path:?} is not inside the project")))?;
|
||||
if job.project_root.join(&rel).exists() {
|
||||
walk_local(job, &rel, 0, &mut local, &mut skipped)?;
|
||||
}
|
||||
rels.push(rel);
|
||||
}
|
||||
let mut remote = connect(job)?;
|
||||
let result = (|| {
|
||||
let mut theirs = Vec::new();
|
||||
for rel in &rels {
|
||||
let mut ignored = 0;
|
||||
walk_remote(job, remote.as_mut(), rel, 0, &mut theirs, &mut ignored)?;
|
||||
}
|
||||
let mut by_path: BTreeMap<String, Difference> = BTreeMap::new();
|
||||
for file in &local {
|
||||
by_path.insert(
|
||||
file.local.clone(),
|
||||
Difference {
|
||||
path: file.local.clone(),
|
||||
status: DiffStatus::LocalOnly,
|
||||
local_size: Some(file.size),
|
||||
remote_size: None,
|
||||
},
|
||||
);
|
||||
}
|
||||
job.monitor.update(|p| {
|
||||
p.phase = "comparing".into();
|
||||
p.files_total = theirs.len();
|
||||
});
|
||||
for file in &theirs {
|
||||
job.cancel.check()?;
|
||||
job.monitor.update(|p| p.file = Some(file.local.clone()));
|
||||
match local.iter().find(|l| l.local == file.local) {
|
||||
None => {
|
||||
by_path.insert(
|
||||
file.local.clone(),
|
||||
Difference {
|
||||
path: file.local.clone(),
|
||||
status: DiffStatus::RemoteOnly,
|
||||
local_size: None,
|
||||
remote_size: Some(file.size),
|
||||
},
|
||||
);
|
||||
}
|
||||
Some(mine) => {
|
||||
let differs = mine.size != file.size
|
||||
|| (mine.size <= MAX_COMPARE
|
||||
&& !same_contents(job, remote.as_mut(), mine)?);
|
||||
if differs {
|
||||
by_path.insert(
|
||||
file.local.clone(),
|
||||
Difference {
|
||||
path: file.local.clone(),
|
||||
status: DiffStatus::Different,
|
||||
local_size: Some(mine.size),
|
||||
remote_size: Some(file.size),
|
||||
},
|
||||
);
|
||||
} else {
|
||||
by_path.remove(&file.local);
|
||||
}
|
||||
}
|
||||
}
|
||||
job.monitor.update(|p| p.files_done += 1);
|
||||
}
|
||||
Ok(by_path.into_values().collect::<Vec<_>>())
|
||||
})();
|
||||
remote.close();
|
||||
let found = result.map_err(|e: DeployError| e.redacted(secret))?;
|
||||
job.monitor.log(
|
||||
Level::Info,
|
||||
format!("compared: {} differing files", found.len()),
|
||||
);
|
||||
Ok(found)
|
||||
}
|
||||
|
||||
/// Uploads the project-relative files `upload` and deletes the files
|
||||
/// `delete` on the server, in one connection. Callers show the plan first
|
||||
/// and pass deletions only after the user confirmed them; the engine
|
||||
/// refuses deletions without that confirmation.
|
||||
///
|
||||
/// # Errors
|
||||
/// Like [`upload`]; a failing file is counted in [`Summary::failed`].
|
||||
pub fn sync(job: &Job, upload: &[String], delete: &[String]) -> Result<Summary> {
|
||||
let secret = job.secret.as_ref().map(Secret::expose);
|
||||
let mut plan = Vec::new();
|
||||
let mut summary = Summary::default();
|
||||
for path in upload {
|
||||
let rel = normalize_relative(path)
|
||||
.ok_or_else(|| DeployError::Invalid(format!("{path:?} is not inside the project")))?;
|
||||
walk_local(job, &rel, 0, &mut plan, &mut summary.skipped)?;
|
||||
}
|
||||
let mut doomed = Vec::new();
|
||||
for path in delete {
|
||||
let rel = normalize_relative(path)
|
||||
.filter(|r| !r.is_empty())
|
||||
.ok_or_else(|| DeployError::Invalid(format!("{path:?} is not a project file")))?;
|
||||
let remote = remote_path(&job.server, &rel)
|
||||
.ok_or_else(|| DeployError::Invalid(format!("{rel} is excluded or not mapped")))?;
|
||||
doomed.push((rel, remote));
|
||||
}
|
||||
job.monitor.update(|p| {
|
||||
p.files_total = plan.len() + doomed.len();
|
||||
p.bytes_total = plan.iter().map(|f| f.size).sum();
|
||||
});
|
||||
let mut remote = connect(job)?;
|
||||
job.monitor.update(|p| p.phase = "transferring".into());
|
||||
let mut made: BTreeSet<String> = BTreeSet::new();
|
||||
let result = (|| {
|
||||
for file in &plan {
|
||||
job.cancel.check()?;
|
||||
job.monitor.update(|p| p.file = Some(file.local.clone()));
|
||||
let outcome =
|
||||
upload_one(job, remote.as_mut(), file, &mut made).map_err(|e| e.redacted(secret));
|
||||
job.monitor.update(|p| p.files_done += 1);
|
||||
match outcome {
|
||||
Ok(()) => {
|
||||
summary.files += 1;
|
||||
summary.bytes += file.size;
|
||||
job.monitor
|
||||
.log(Level::Info, format!("{} → {}", file.local, file.remote));
|
||||
}
|
||||
Err(e) if is_fatal(&e) => return Err(e),
|
||||
Err(e) => {
|
||||
summary.failed += 1;
|
||||
job.monitor
|
||||
.log(Level::Error, format!("{}: {e}", file.local));
|
||||
}
|
||||
}
|
||||
}
|
||||
for (rel, path) in &doomed {
|
||||
job.cancel.check()?;
|
||||
job.monitor.update(|p| p.file = Some(rel.clone()));
|
||||
let outcome = remote.remove_file(path).map_err(|e| e.redacted(secret));
|
||||
job.monitor.update(|p| p.files_done += 1);
|
||||
match outcome {
|
||||
Ok(()) => {
|
||||
summary.deleted += 1;
|
||||
job.monitor
|
||||
.log(Level::Warning, format!("deleted {path} on the server"));
|
||||
}
|
||||
Err(e) if is_fatal(&e) => return Err(e),
|
||||
Err(e) => {
|
||||
summary.failed += 1;
|
||||
job.monitor.log(Level::Error, format!("{rel}: {e}"));
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
})();
|
||||
remote.close();
|
||||
result.map(|()| summary)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
|
|
|||
|
|
@ -9,13 +9,16 @@
|
|||
//! - [`remote`]: the operations every transport offers (the extension
|
||||
//! point for the remote browser and compare of D2), and the folder
|
||||
//! transport; `sftp` and `ftp` are the network ones.
|
||||
//! - [`job`]: test connection, upload and download with progress, log and
|
||||
//! cancel.
|
||||
//! - [`job`]: test connection, upload, download, compare and sync with
|
||||
//! progress, log and cancel.
|
||||
//! - [`browse`]: the remote browser's list, read, write, rename, delete and
|
||||
//! new folder.
|
||||
//!
|
||||
//! Pure Rust: `russh` and `russh-sftp` for SFTP, `suppaftp` with `rustls`
|
||||
//! for FTP and FTPS, `ring` for cryptography, `keyring` for the keychain.
|
||||
//! No credential is ever written to a file, a log or an error message.
|
||||
|
||||
pub mod browse;
|
||||
pub mod config;
|
||||
pub mod error;
|
||||
mod ftp;
|
||||
|
|
@ -31,6 +34,6 @@ pub mod testserver;
|
|||
|
||||
pub use config::{AuthKind, Deployment, Mapping, Server, ServerKind};
|
||||
pub use error::{DeployError, Result};
|
||||
pub use job::{Job, Level, LogLine, Monitor, Progress, Summary};
|
||||
pub use job::{DiffStatus, Difference, Job, Level, LogLine, Monitor, Progress, Summary};
|
||||
pub use remote::Cancel;
|
||||
pub use secrets::{Keychain, MemoryStore, Secret, SecretStore};
|
||||
|
|
|
|||
|
|
@ -86,7 +86,7 @@ pub fn is_excluded(server: &Server, path: &str) -> bool {
|
|||
|
||||
/// Joins remote path pieces with `/`, skipping empty ones. Keeps a leading
|
||||
/// `/` of the first non-empty piece.
|
||||
fn join(pieces: &[&str]) -> String {
|
||||
pub(crate) fn join(pieces: &[&str]) -> String {
|
||||
let mut out = String::new();
|
||||
for piece in pieces.iter().filter(|p| !p.is_empty()) {
|
||||
if out.is_empty() {
|
||||
|
|
|
|||
|
|
@ -115,6 +115,25 @@ pub trait Remote {
|
|||
/// [`DeployError::Transfer`] when the server refuses.
|
||||
fn list(&mut self, path: &str) -> Result<Vec<Entry>>;
|
||||
|
||||
/// Removes the file `path`.
|
||||
///
|
||||
/// # Errors
|
||||
/// [`DeployError::Transfer`] when the server refuses.
|
||||
fn remove_file(&mut self, path: &str) -> Result<()>;
|
||||
|
||||
/// Removes the empty folder `path`.
|
||||
///
|
||||
/// # Errors
|
||||
/// [`DeployError::Transfer`] when the server refuses (also when the
|
||||
/// folder is not empty).
|
||||
fn remove_dir(&mut self, path: &str) -> Result<()>;
|
||||
|
||||
/// Renames or moves `from` to `to` on the server.
|
||||
///
|
||||
/// # Errors
|
||||
/// [`DeployError::Transfer`] when the server refuses.
|
||||
fn rename(&mut self, from: &str, to: &str) -> Result<()>;
|
||||
|
||||
/// Ends the session politely; failures are ignored because the
|
||||
/// connection is dropped anyway.
|
||||
fn close(&mut self);
|
||||
|
|
@ -247,6 +266,27 @@ impl Remote for Folder {
|
|||
Ok(entries)
|
||||
}
|
||||
|
||||
fn remove_file(&mut self, path: &str) -> Result<()> {
|
||||
let path = self.resolve(path)?;
|
||||
std::fs::remove_file(&path).map_err(|e| io_error(&path, &e))
|
||||
}
|
||||
|
||||
fn remove_dir(&mut self, path: &str) -> Result<()> {
|
||||
let path = self.resolve(path)?;
|
||||
if path == self.root {
|
||||
return Err(DeployError::Invalid(
|
||||
"the deployment folder itself cannot be removed".into(),
|
||||
));
|
||||
}
|
||||
std::fs::remove_dir(&path).map_err(|e| io_error(&path, &e))
|
||||
}
|
||||
|
||||
fn rename(&mut self, from: &str, to: &str) -> Result<()> {
|
||||
let from = self.resolve(from)?;
|
||||
let to = self.resolve(to)?;
|
||||
std::fs::rename(&from, &to).map_err(|e| io_error(&from, &e))
|
||||
}
|
||||
|
||||
fn close(&mut self) {}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -445,6 +445,24 @@ impl Remote for Sftp {
|
|||
Ok(entries)
|
||||
}
|
||||
|
||||
fn remove_file(&mut self, path: &str) -> Result<()> {
|
||||
self.run("removing a file", self.sftp.remove_file(path.to_string()))?
|
||||
.map_err(|e| sftp_error("cannot remove", path, &e))
|
||||
}
|
||||
|
||||
fn remove_dir(&mut self, path: &str) -> Result<()> {
|
||||
self.run("removing a folder", self.sftp.remove_dir(path.to_string()))?
|
||||
.map_err(|e| sftp_error("cannot remove the folder", path, &e))
|
||||
}
|
||||
|
||||
fn rename(&mut self, from: &str, to: &str) -> Result<()> {
|
||||
self.run(
|
||||
"renaming",
|
||||
self.sftp.rename(from.to_string(), to.to_string()),
|
||||
)?
|
||||
.map_err(|e| sftp_error(&format!("cannot rename to {to}:"), from, &e))
|
||||
}
|
||||
|
||||
/// Closes the SFTP channel and the connection. Failures are ignored:
|
||||
/// the connection is dropped right after.
|
||||
fn close(&mut self) {
|
||||
|
|
|
|||
|
|
@ -6,8 +6,8 @@
|
|||
//! The SFTP server has a fixed Ed25519 host key (from a seed) and accepts
|
||||
//! one user with one password, or one public key. The FTP server speaks
|
||||
//! enough of RFC 959 for Lutra's client: USER, PASS, SYST, FEAT, TYPE,
|
||||
//! PWD, CWD, MKD, SIZE, PASV, STOR, RETR, LIST, NOOP, QUIT. Both stop when
|
||||
//! dropped.
|
||||
//! PWD, CWD, MKD, RMD, DELE, RNFR, RNTO, SIZE, PASV, STOR, RETR, LIST,
|
||||
//! NOOP, QUIT. Both stop when dropped.
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::io::{BufRead, BufReader, Read, Seek, SeekFrom, Write};
|
||||
|
|
@ -465,6 +465,16 @@ impl russh_sftp::server::Handler for SftpHandler {
|
|||
Ok(ok(id))
|
||||
}
|
||||
|
||||
async fn rename(
|
||||
&mut self,
|
||||
id: u32,
|
||||
oldpath: String,
|
||||
newpath: String,
|
||||
) -> Result<Status, Self::Error> {
|
||||
std::fs::rename(self.path(&oldpath)?, self.path(&newpath)?).map_err(|e| io_status(&e))?;
|
||||
Ok(ok(id))
|
||||
}
|
||||
|
||||
async fn rmdir(&mut self, id: u32, path: String) -> Result<Status, Self::Error> {
|
||||
std::fs::remove_dir(self.path(&path)?).map_err(|e| io_status(&e))?;
|
||||
Ok(ok(id))
|
||||
|
|
@ -532,6 +542,7 @@ fn ftp_session(stream: TcpStream, root: &Path, user: &str, password: &str) -> st
|
|||
let mut signed_in = false;
|
||||
let mut cwd = String::from("/");
|
||||
let mut passive: Option<TcpListener> = None;
|
||||
let mut rename_from: Option<PathBuf> = None;
|
||||
loop {
|
||||
let mut line = String::new();
|
||||
if reader.read_line(&mut line)? == 0 {
|
||||
|
|
@ -579,6 +590,27 @@ fn ftp_session(stream: TcpStream, root: &Path, user: &str, password: &str) -> st
|
|||
Some(Ok(())) => send(&format!("257 \"{arg}\" created"))?,
|
||||
_ => send("550 Cannot create")?,
|
||||
},
|
||||
"RMD" => match path(arg).map(std::fs::remove_dir) {
|
||||
Some(Ok(())) => send("250 Removed")?,
|
||||
_ => send("550 Cannot remove")?,
|
||||
},
|
||||
"DELE" => match path(arg).map(std::fs::remove_file) {
|
||||
Some(Ok(())) => send("250 Deleted")?,
|
||||
_ => send("550 Cannot delete")?,
|
||||
},
|
||||
"RNFR" => match path(arg).filter(|p| p.exists()) {
|
||||
Some(p) => {
|
||||
rename_from = Some(p);
|
||||
send("350 Ready for RNTO")?;
|
||||
}
|
||||
None => send("550 No such file")?,
|
||||
},
|
||||
"RNTO" => match (rename_from.take(), path(arg)) {
|
||||
(Some(from), Some(to)) if std::fs::rename(&from, &to).is_ok() => {
|
||||
send("250 Renamed")?
|
||||
}
|
||||
_ => send("550 Cannot rename")?,
|
||||
},
|
||||
"SIZE" => match path(arg).and_then(|p| std::fs::metadata(p).ok()) {
|
||||
Some(m) if m.is_file() => send(&format!("213 {}", m.len()))?,
|
||||
_ => send("550 No such file")?,
|
||||
|
|
|
|||
170
crates/deploy/tests/remote.rs
Normal file
170
crates/deploy/tests/remote.rs
Normal file
|
|
@ -0,0 +1,170 @@
|
|||
//! The remote browser, compare and sync (D2) against the local SFTP and
|
||||
//! FTP test servers (127.0.0.1 only): list, read (text, binary, missing),
|
||||
//! write, rename, new folder, recursive delete (never the root), compare a
|
||||
//! folder, and sync with uploads and confirmed deletions.
|
||||
|
||||
#![allow(
|
||||
clippy::unwrap_used,
|
||||
clippy::expect_used,
|
||||
clippy::panic,
|
||||
clippy::indexing_slicing
|
||||
)]
|
||||
|
||||
use std::path::Path;
|
||||
|
||||
use lutra_deploy::browse::{self, RemoteText, Target};
|
||||
use lutra_deploy::known_hosts::{self, HostKey};
|
||||
use lutra_deploy::testserver::{FtpServer, SftpServer};
|
||||
use lutra_deploy::{AuthKind, Cancel, DiffStatus, Job, Monitor, Secret, Server, ServerKind, job};
|
||||
|
||||
const PASSWORD: &str = "Otter-Browse-Pa55!";
|
||||
|
||||
fn server(kind: ServerKind, port: u16) -> Server {
|
||||
Server {
|
||||
id: "t".into(),
|
||||
name: "Test".into(),
|
||||
kind,
|
||||
host: "127.0.0.1".into(),
|
||||
port: Some(port),
|
||||
user: "lutra".into(),
|
||||
root: "/www".into(),
|
||||
auth: AuthKind::Password,
|
||||
key_path: None,
|
||||
mappings: vec![],
|
||||
excluded: vec!["*.log".into()],
|
||||
upload_on_save: false,
|
||||
}
|
||||
}
|
||||
|
||||
fn job(server: Server, project: &Path, known: &Path) -> Job {
|
||||
Job {
|
||||
server,
|
||||
project_root: project.to_path_buf(),
|
||||
secret: Some(Secret::new(PASSWORD.into()).unwrap()),
|
||||
known_hosts: known.to_path_buf(),
|
||||
cancel: Cancel::new(),
|
||||
monitor: Monitor::new(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Runs the whole browser, compare and sync scenario against one server.
|
||||
fn scenario(j: &Job, remote: &Path) {
|
||||
let project = &j.project_root;
|
||||
std::fs::create_dir_all(project.join("src")).unwrap();
|
||||
std::fs::write(project.join("index.php"), "<?php // same").unwrap();
|
||||
std::fs::write(project.join("src/a.php"), "<?php // local").unwrap();
|
||||
std::fs::write(project.join("src/new.php"), "<?php // new").unwrap();
|
||||
std::fs::write(project.join("src/x.log"), "ignored").unwrap();
|
||||
let www = remote.join("www");
|
||||
std::fs::create_dir_all(www.join("src")).unwrap();
|
||||
std::fs::write(www.join("index.php"), "<?php // same").unwrap();
|
||||
std::fs::write(www.join("src/a.php"), "<?php // SERVER").unwrap();
|
||||
std::fs::write(www.join("src/old.php"), "<?php // old").unwrap();
|
||||
std::fs::write(www.join("blob.bin"), [0u8, 159, 146, 150]).unwrap();
|
||||
|
||||
let names: Vec<String> = browse::list(j, "")
|
||||
.unwrap()
|
||||
.into_iter()
|
||||
.map(|e| e.name)
|
||||
.collect();
|
||||
assert_eq!(names, ["src", "blob.bin", "index.php"]);
|
||||
let a = browse::read(j, &Target::Mapped("src/a.php".into())).unwrap();
|
||||
assert_eq!(a.text.as_deref(), Some("<?php // SERVER"));
|
||||
assert!(
|
||||
browse::read(j, &Target::Browser("blob.bin".into()))
|
||||
.unwrap()
|
||||
.binary
|
||||
);
|
||||
assert_eq!(
|
||||
browse::read(j, &Target::Browser("nope.php".into())).unwrap(),
|
||||
RemoteText::default()
|
||||
);
|
||||
assert!(browse::read(j, &Target::Mapped("src/x.log".into())).is_err());
|
||||
|
||||
browse::write(j, "edited/e.php", b"<?php // edited").unwrap();
|
||||
assert_eq!(
|
||||
std::fs::read_to_string(www.join("edited/e.php")).unwrap(),
|
||||
"<?php // edited"
|
||||
);
|
||||
browse::rename(j, "edited/e.php", "edited/f.php").unwrap();
|
||||
assert!(www.join("edited/f.php").is_file());
|
||||
browse::mkdir(j, "edited/deep/er").unwrap();
|
||||
assert!(www.join("edited/deep/er").is_dir());
|
||||
assert_eq!(browse::delete(j, "edited").unwrap(), 4);
|
||||
assert!(!www.join("edited").exists());
|
||||
assert!(browse::delete(j, "").is_err());
|
||||
assert!(browse::list(j, "../..").is_err());
|
||||
|
||||
let diffs = job::compare(j, &[String::new()]).unwrap();
|
||||
let seen: Vec<(String, DiffStatus)> =
|
||||
diffs.iter().map(|d| (d.path.clone(), d.status)).collect();
|
||||
assert_eq!(
|
||||
seen,
|
||||
[
|
||||
("blob.bin".to_string(), DiffStatus::RemoteOnly),
|
||||
("src/a.php".to_string(), DiffStatus::Different),
|
||||
("src/new.php".to_string(), DiffStatus::LocalOnly),
|
||||
("src/old.php".to_string(), DiffStatus::RemoteOnly),
|
||||
]
|
||||
);
|
||||
|
||||
let summary = job::sync(
|
||||
j,
|
||||
&["src/a.php".into(), "src/new.php".into()],
|
||||
&["src/old.php".into()],
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!((summary.files, summary.deleted, summary.failed), (2, 1, 0));
|
||||
assert!(!www.join("src/old.php").exists());
|
||||
assert!(
|
||||
www.join("blob.bin").exists(),
|
||||
"only confirmed files are deleted"
|
||||
);
|
||||
let after = job::compare(j, &["src".into()]).unwrap();
|
||||
assert!(after.is_empty(), "{after:?}");
|
||||
let text = j
|
||||
.monitor
|
||||
.take_lines()
|
||||
.into_iter()
|
||||
.map(|l| l.message)
|
||||
.collect::<Vec<_>>()
|
||||
.join("\n");
|
||||
assert!(text.contains("deleted /www/src/old.php"));
|
||||
assert!(!text.contains(PASSWORD));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sftp_browse_compare_and_sync() {
|
||||
let remote = tempfile::tempdir().unwrap();
|
||||
let sftp = SftpServer::start(remote.path(), "lutra", Some(PASSWORD), None, 21).unwrap();
|
||||
let project = tempfile::tempdir().unwrap();
|
||||
let state = tempfile::tempdir().unwrap();
|
||||
let known = state.path().join("known_hosts");
|
||||
let s = server(ServerKind::Sftp, sftp.addr.port());
|
||||
known_hosts::trust(
|
||||
&known,
|
||||
&HostKey {
|
||||
host: s.address(),
|
||||
algorithm: sftp.algorithm.clone(),
|
||||
fingerprint: sftp.fingerprint.clone(),
|
||||
},
|
||||
)
|
||||
.unwrap();
|
||||
scenario(&job(s, project.path(), &known), remote.path());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ftp_browse_compare_and_sync() {
|
||||
let remote = tempfile::tempdir().unwrap();
|
||||
let ftp = FtpServer::start(remote.path(), "lutra", PASSWORD).unwrap();
|
||||
let project = tempfile::tempdir().unwrap();
|
||||
let known = project.path().join("unused");
|
||||
scenario(
|
||||
&job(
|
||||
server(ServerKind::Ftp, ftp.addr.port()),
|
||||
project.path(),
|
||||
&known,
|
||||
),
|
||||
remote.path(),
|
||||
);
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue