D1: lutra-deploy crate: SFTP, FTP/FTPS and folder transports, keychain secrets, known hosts, path mappings

Pure Rust (D27): russh + russh-sftp with ring, suppaftp with rustls/ring,
keyring (Windows Credential Manager, macOS Keychain). Lutra's own
known-hosts file, refuse on mismatch. In-process SFTP and FTP test
servers behind the testserver feature.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
danny 2026-11-02 19:09:17 +01:00
parent 33a7185d75
commit 0644b044f2
16 changed files with 5250 additions and 13 deletions

627
crates/deploy/src/job.rs Normal file
View file

@ -0,0 +1,627 @@
//! Jobs: test a connection, upload or download project paths.
//!
//! A job first plans (walks the local folders or lists the remote ones and
//! maps every file), then connects once and transfers file by file. Its
//! [`Monitor`] holds the progress and the new log lines; the engine reads
//! it from another thread. A failing file is logged and the job goes on; a
//! lost connection, a refused sign-in or a cancel ends it.
//!
//! Folder walks skip symbolic links to folders and the version-control
//! folders `.git`, `.svn` and `.hg`, which are never deployed.
use std::collections::BTreeSet;
use std::fs::File;
use std::path::Path;
use std::sync::{Arc, Mutex, PoisonError};
use serde::Serialize;
use crate::config::{Server, ServerKind, io_error};
use crate::error::{DeployError, Result};
use crate::mapping::{is_excluded, normalize_relative, remote_path};
use crate::remote::{Cancel, Folder, Remote};
use crate::secrets::Secret;
/// Most files one job plans.
pub const MAX_FILES: usize = 200_000;
/// Deepest folder nesting a job follows.
const MAX_DEPTH: usize = 64;
/// Folders never deployed.
const VCS_DIRS: [&str; 3] = [".git", ".svn", ".hg"];
/// Severity of a log line.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
#[serde(rename_all = "lowercase")]
pub enum Level {
/// Something worked.
Info,
/// Something was skipped.
Warning,
/// Something failed.
Error,
}
/// One line of the deployment log.
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
pub struct LogLine {
/// Severity.
pub level: Level,
/// The text; never contains a secret.
pub message: String,
}
/// Progress of a running job.
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct Progress {
/// `connecting`, `planning`, `transferring`.
pub phase: String,
/// The file being transferred, project-relative.
#[serde(skip_serializing_if = "Option::is_none")]
pub file: Option<String>,
/// Files finished, successful or not.
pub files_done: usize,
/// Files planned.
pub files_total: usize,
/// Bytes transferred.
pub bytes_done: u64,
/// Bytes planned (uploads only; 0 when unknown).
pub bytes_total: u64,
}
/// What a job shares with its observer.
#[derive(Debug, Default)]
struct Shared {
/// The progress.
progress: Progress,
/// Log lines not taken yet.
lines: Vec<LogLine>,
}
/// A job's progress and log, readable from another thread. Clones share
/// the state.
#[derive(Debug, Clone, Default)]
pub struct Monitor(Arc<Mutex<Shared>>);
impl Monitor {
/// A fresh monitor.
pub fn new() -> Self {
Self::default()
}
/// The current progress.
pub fn progress(&self) -> Progress {
self.0
.lock()
.unwrap_or_else(PoisonError::into_inner)
.progress
.clone()
}
/// Takes the log lines written since the last call.
pub fn take_lines(&self) -> Vec<LogLine> {
std::mem::take(&mut self.0.lock().unwrap_or_else(PoisonError::into_inner).lines)
}
/// Changes the progress.
fn update(&self, change: impl FnOnce(&mut Progress)) {
change(
&mut self
.0
.lock()
.unwrap_or_else(PoisonError::into_inner)
.progress,
);
}
/// Adds a log line.
fn log(&self, level: Level, message: String) {
self.0
.lock()
.unwrap_or_else(PoisonError::into_inner)
.lines
.push(LogLine { level, message });
}
}
/// Everything a job needs.
#[derive(Debug, Clone)]
pub struct Job {
/// The server.
pub server: Server,
/// The project root on disk.
pub project_root: std::path::PathBuf,
/// The password or passphrase from the keychain, when the server
/// needs one and it is stored.
pub secret: Option<Secret>,
/// Lutra's known-hosts file.
pub known_hosts: std::path::PathBuf,
/// Stops the job.
pub cancel: Cancel,
/// Progress and log.
pub monitor: Monitor,
}
/// What a finished upload or download did.
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct Summary {
/// Files transferred.
pub files: usize,
/// Bytes transferred.
pub bytes: u64,
/// Files skipped: excluded, not mapped, or missing on the server.
pub skipped: usize,
/// Files that failed.
pub failed: usize,
}
/// One planned file.
#[derive(Debug, Clone)]
struct Planned {
/// Project-relative path.
local: String,
/// Remote path.
remote: String,
/// Size, when known.
size: u64,
}
/// Connects to the job's server.
///
/// # Errors
/// What the transport's `connect` returns, with the secret redacted.
pub fn connect(job: &Job) -> Result<Box<dyn Remote>> {
job.monitor.update(|p| p.phase = "connecting".into());
let secret = job.secret.as_ref();
let remote: Result<Box<dyn Remote>> = match job.server.kind {
ServerKind::Sftp => {
crate::sftp::Sftp::connect(&job.server, secret, &job.known_hosts, &job.cancel)
.map(|s| Box::new(s) as Box<dyn Remote>)
}
ServerKind::Ftp | ServerKind::Ftps => {
crate::ftp::Ftp::connect(&job.server, secret, &job.cancel)
.map(|s| Box::new(s) as Box<dyn Remote>)
}
ServerKind::Folder => {
Folder::open(Path::new(&job.server.root)).map(|s| Box::new(s) as Box<dyn Remote>)
}
};
remote.map_err(|e| e.redacted(secret.map(Secret::expose)))
}
/// Connects, signs in and checks the root folder. Returns a sentence for
/// the user.
///
/// # Errors
/// What [`connect`] returns.
pub fn test_connection(job: &Job) -> Result<String> {
let mut remote = connect(job)?;
let root = if job.server.kind == ServerKind::Folder {
String::new()
} else {
job.server.root.clone()
};
let kind = remote
.kind(&root)
.map_err(|e| e.redacted(job.secret.as_ref().map(Secret::expose)));
remote.close();
let kind = kind?;
let name = &job.server.name;
let shown = display_root(&root);
let message = match kind {
_ if job.server.kind == ServerKind::Folder => {
format!("Connected to the folder {}.", job.server.root)
}
Some(true) => format!("Connected to {name}; the root folder {shown} exists."),
Some(false) => {
format!("Connected to {name}, but the root {shown} is a file, not a folder.")
}
None => format!(
"Connected to {name}; the root folder {shown} does not exist yet and is created by the first upload."
),
};
job.monitor.log(Level::Info, message.clone());
Ok(message)
}
/// The root as shown to the user.
fn display_root(root: &str) -> &str {
if root.is_empty() {
"(login folder)"
} else {
root
}
}
/// Adds the files under the project-relative `rel` to `plan`.
fn walk_local(
job: &Job,
rel: &str,
depth: usize,
plan: &mut Vec<Planned>,
skipped: &mut usize,
) -> Result<()> {
job.cancel.check()?;
let path = if rel.is_empty() {
job.project_root.clone()
} else {
job.project_root.join(rel)
};
let meta = std::fs::symlink_metadata(&path).map_err(|e| io_error(&path, &e))?;
let is_dir = meta.is_dir() || (meta.file_type().is_symlink() && depth == 0 && path.is_dir());
if !is_dir {
let size = std::fs::metadata(&path)
.map_err(|e| io_error(&path, &e))?
.len();
match remote_path(&job.server, rel) {
Some(remote) if !rel.is_empty() => plan.push(Planned {
local: rel.to_string(),
remote,
size,
}),
_ => {
*skipped += 1;
job.monitor.log(
Level::Warning,
format!("{rel}: skipped (excluded or not mapped)"),
);
}
}
if plan.len() > MAX_FILES {
return Err(DeployError::Invalid(format!(
"more than {MAX_FILES} files; upload a smaller folder"
)));
}
return Ok(());
}
if depth > MAX_DEPTH || (!rel.is_empty() && is_excluded(&job.server, rel)) {
return Ok(());
}
let mut names: Vec<String> = std::fs::read_dir(&path)
.map_err(|e| io_error(&path, &e))?
.filter_map(|e| e.ok())
.filter(|e| {
e.file_type()
.map(|t| !(t.is_symlink() && e.path().is_dir()))
.unwrap_or(false)
})
.map(|e| e.file_name().to_string_lossy().into_owned())
.filter(|n| !VCS_DIRS.contains(&n.as_str()))
.collect();
names.sort();
for name in names {
let child = if rel.is_empty() {
name
} else {
format!("{rel}/{name}")
};
walk_local(job, &child, depth + 1, plan, skipped)?;
}
Ok(())
}
/// The parent folder of a remote path; empty for a top-level one.
fn parent(remote: &str) -> &str {
match remote.rfind('/') {
Some(0) => "/",
Some(i) => remote.get(..i).unwrap_or_default(),
None => "",
}
}
/// Whether an error ends the whole job instead of one file.
fn is_fatal(e: &DeployError) -> bool {
!matches!(e, DeployError::Transfer(_))
}
/// Uploads the project-relative `paths` (files or folders).
///
/// # Errors
/// [`DeployError::Invalid`] for a path outside the project, otherwise
/// what [`connect`] returns, or [`DeployError::Cancelled`]. Failing files
/// are counted in [`Summary::failed`] instead.
pub fn upload(job: &Job, paths: &[String]) -> Result<Summary> {
let secret = job.secret.as_ref().map(Secret::expose);
job.monitor.update(|p| p.phase = "planning".into());
let mut plan = Vec::new();
let mut summary = Summary::default();
for path in paths {
let rel = normalize_relative(path)
.ok_or_else(|| DeployError::Invalid(format!("{path:?} is not inside the project")))?;
walk_local(job, &rel, 0, &mut plan, &mut summary.skipped)?;
}
let total: u64 = plan.iter().map(|p| p.size).sum();
job.monitor.update(|p| {
p.files_total = plan.len();
p.bytes_total = total;
});
if plan.is_empty() {
job.monitor.log(Level::Warning, "nothing to upload".into());
return Ok(summary);
}
let mut remote = connect(job)?;
job.monitor.update(|p| p.phase = "transferring".into());
let mut made: BTreeSet<String> = BTreeSet::new();
let result = (|| {
for file in &plan {
job.cancel.check()?;
job.monitor.update(|p| p.file = Some(file.local.clone()));
let outcome =
upload_one(job, remote.as_mut(), file, &mut made).map_err(|e| e.redacted(secret));
job.monitor.update(|p| p.files_done += 1);
match outcome {
Ok(()) => {
summary.files += 1;
summary.bytes += file.size;
job.monitor
.log(Level::Info, format!("{} → {}", file.local, file.remote));
}
Err(e) if is_fatal(&e) => return Err(e),
Err(e) => {
summary.failed += 1;
job.monitor
.log(Level::Error, format!("{}: {e}", file.local));
}
}
}
Ok(())
})();
remote.close();
result.map(|()| summary)
}
/// Uploads one file, creating its remote folder first.
fn upload_one(
job: &Job,
remote: &mut dyn Remote,
file: &Planned,
made: &mut BTreeSet<String>,
) -> Result<()> {
let dir = parent(&file.remote).to_string();
if !dir.is_empty() && dir != "/" && !made.contains(&dir) {
remote.mkdir_all(&dir)?;
made.insert(dir);
}
let path = job.project_root.join(&file.local);
let mut data = File::open(&path).map_err(|e| io_error(&path, &e))?;
let monitor = job.monitor.clone();
let cancel = job.cancel.clone();
remote.put(&file.remote, &mut data, &mut |n| {
monitor.update(|p| p.bytes_done = p.bytes_done.saturating_add(n));
cancel.check()
})
}
/// Adds the remote files under the project-relative `rel` to `plan`.
fn walk_remote(
job: &Job,
remote: &mut dyn Remote,
rel: &str,
depth: usize,
plan: &mut Vec<Planned>,
skipped: &mut usize,
) -> Result<()> {
job.cancel.check()?;
let Some(path) = remote_path(&job.server, rel) else {
*skipped += 1;
job.monitor.log(
Level::Warning,
format!("{rel}: skipped (excluded or not mapped)"),
);
return Ok(());
};
match remote.kind(&path)? {
None => {
*skipped += 1;
job.monitor
.log(Level::Warning, format!("{rel}: not on the server ({path})"));
}
Some(false) if !rel.is_empty() => plan.push(Planned {
local: rel.to_string(),
remote: path,
size: 0,
}),
Some(false) => {
*skipped += 1;
}
Some(true) if depth <= MAX_DEPTH => {
for entry in remote.list(&path)? {
if VCS_DIRS.contains(&entry.name.as_str()) {
continue;
}
let child = if rel.is_empty() {
entry.name
} else {
format!("{rel}/{}", entry.name)
};
walk_remote(job, remote, &child, depth + 1, plan, skipped)?;
if plan.len() > MAX_FILES {
return Err(DeployError::Invalid(format!(
"more than {MAX_FILES} files; download a smaller folder"
)));
}
}
}
Some(true) => {}
}
Ok(())
}
/// Downloads the project-relative `paths` (files or folders) from the
/// server into the project, replacing local files.
///
/// # Errors
/// Like [`upload`].
pub fn download(job: &Job, paths: &[String]) -> Result<Summary> {
let secret = job.secret.as_ref().map(Secret::expose);
let mut rels = Vec::new();
for path in paths {
rels.push(
normalize_relative(path).ok_or_else(|| {
DeployError::Invalid(format!("{path:?} is not inside the project"))
})?,
);
}
let mut remote = connect(job)?;
let mut summary = Summary::default();
let result = (|| {
job.monitor.update(|p| p.phase = "planning".into());
let mut plan = Vec::new();
for rel in &rels {
walk_remote(
job,
remote.as_mut(),
rel,
0,
&mut plan,
&mut summary.skipped,
)
.map_err(|e| e.redacted(secret))?;
}
job.monitor.update(|p| {
p.files_total = plan.len();
p.phase = "transferring".into();
});
for file in &plan {
job.cancel.check()?;
job.monitor.update(|p| p.file = Some(file.local.clone()));
let outcome = download_one(job, remote.as_mut(), file).map_err(|e| e.redacted(secret));
job.monitor.update(|p| p.files_done += 1);
match outcome {
Ok(bytes) => {
summary.files += 1;
summary.bytes += bytes;
job.monitor
.log(Level::Info, format!("{} ← {}", file.local, file.remote));
}
Err(e) if is_fatal(&e) => return Err(e),
Err(e) => {
summary.failed += 1;
job.monitor
.log(Level::Error, format!("{}: {e}", file.local));
}
}
}
Ok(())
})();
remote.close();
result.map(|()| summary)
}
/// Downloads one file through a temporary file next to it. Returns its
/// size. A temporary file that cannot be removed after a failure is left;
/// the next download of the same file replaces it.
fn download_one(job: &Job, remote: &mut dyn Remote, file: &Planned) -> Result<u64> {
let target = job.project_root.join(&file.local);
if let Some(dir) = target.parent() {
std::fs::create_dir_all(dir).map_err(|e| io_error(dir, &e))?;
}
let temp = target.with_extension("lutra-download");
let monitor = job.monitor.clone();
let cancel = job.cancel.clone();
let mut bytes = 0u64;
let result = (|| {
let mut out = File::create(&temp).map_err(|e| io_error(&temp, &e))?;
remote.get(&file.remote, &mut out, &mut |n| {
bytes = bytes.saturating_add(n);
monitor.update(|p| p.bytes_done = p.bytes_done.saturating_add(n));
cancel.check()
})?;
drop(out);
std::fs::rename(&temp, &target).map_err(|e| io_error(&target, &e))
})();
if result.is_err() {
let _ = std::fs::remove_file(&temp);
}
result.map(|()| bytes)
}
#[cfg(test)]
mod tests {
use super::*;
use crate::config::{AuthKind, Mapping};
fn folder_job(project: &Path, target: &Path) -> Job {
Job {
server: Server {
id: "f".into(),
name: "Mounted".into(),
kind: ServerKind::Folder,
host: String::new(),
port: None,
user: String::new(),
root: target.display().to_string(),
auth: AuthKind::Password,
key_path: None,
mappings: vec![Mapping {
local: String::new(),
remote: "site".into(),
}],
excluded: vec!["*.log".into()],
upload_on_save: false,
},
project_root: project.to_path_buf(),
secret: None,
known_hosts: project.join("kh"),
cancel: Cancel::new(),
monitor: Monitor::new(),
}
}
#[test]
fn parents() {
assert_eq!(parent("/a/b"), "/a");
assert_eq!(parent("/a"), "/");
assert_eq!(parent("a/b"), "a");
assert_eq!(parent("a"), "");
}
#[test]
fn upload_and_download_through_a_folder() {
let project = tempfile::tempdir().unwrap();
let target = tempfile::tempdir().unwrap();
std::fs::create_dir_all(project.path().join("src/sub")).unwrap();
std::fs::create_dir_all(project.path().join(".git")).unwrap();
std::fs::write(project.path().join("src/a.php"), "<?php // a").unwrap();
std::fs::write(project.path().join("src/sub/b.php"), "<?php // b").unwrap();
std::fs::write(project.path().join("src/x.log"), "log").unwrap();
std::fs::write(project.path().join(".git/HEAD"), "ref").unwrap();
let job = folder_job(project.path(), target.path());
let summary = upload(&job, &[String::new()]).unwrap();
assert_eq!(summary.files, 2);
assert_eq!(summary.failed, 0);
assert_eq!(
std::fs::read_to_string(target.path().join("site/src/sub/b.php")).unwrap(),
"<?php // b"
);
assert!(!target.path().join("site/src/x.log").exists());
assert!(!target.path().join("site/.git").exists());
let progress = job.monitor.progress();
assert_eq!(progress.files_done, 2);
assert_eq!(progress.bytes_done, 20);
assert!(
job.monitor
.take_lines()
.iter()
.any(|l| l.message.contains("src/a.php → src/a.php")
|| l.message.contains("src/a.php →"))
);
std::fs::write(target.path().join("site/src/a.php"), "<?php // server").unwrap();
let summary = download(&job, &["src".into()]).unwrap();
assert_eq!(summary.files, 2);
assert_eq!(
std::fs::read_to_string(project.path().join("src/a.php")).unwrap(),
"<?php // server"
);
assert!(upload(&job, &["../x".into()]).is_err());
let summary = download(&job, &["missing.php".into()]).unwrap();
assert_eq!(summary.skipped, 1);
let msg = test_connection(&job).unwrap();
assert!(msg.contains("Connected"), "{msg}");
job.cancel.cancel();
assert_eq!(upload(&job, &["src".into()]), Err(DeployError::Cancelled));
}
}