D1: lutra-deploy crate: SFTP, FTP/FTPS and folder transports, keychain secrets, known hosts, path mappings

Pure Rust (D27): russh + russh-sftp with ring, suppaftp with rustls/ring,
keyring (Windows Credential Manager, macOS Keychain). Lutra's own
known-hosts file, refuse on mismatch. In-process SFTP and FTP test
servers behind the testserver feature.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
danny 2026-11-02 19:09:17 +01:00
parent 33a7185d75
commit 0644b044f2
16 changed files with 5250 additions and 13 deletions

294
crates/deploy/src/remote.rs Normal file
View file

@ -0,0 +1,294 @@
//! The operations every transport offers, cancellation and progress, and
//! the folder transport.
//!
//! Remote paths are `/`-separated strings as [`crate::mapping`] builds
//! them. Directory listings come from the server and are untrusted: names
//! with `/`, `\`, control characters, `.` or `..` are dropped by
//! [`safe_name`] before they become local paths.
use std::io::{Read, Write};
use std::path::{Path, PathBuf};
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, Ordering};
use crate::config::io_error;
use crate::error::{DeployError, Result};
use crate::mapping::normalize_relative;
/// Size of one transfer chunk; progress and cancellation are checked
/// between chunks.
pub const CHUNK: usize = 32 * 1024;
/// Most entries read from one remote folder.
pub const MAX_LISTING: usize = 100_000;
/// Stops a running job. Clones share the flag.
#[derive(Debug, Clone, Default)]
pub struct Cancel(Arc<AtomicBool>);
impl Cancel {
/// A flag that is not set.
pub fn new() -> Self {
Self::default()
}
/// Asks the job to stop.
pub fn cancel(&self) {
self.0.store(true, Ordering::SeqCst);
}
/// Whether the job was asked to stop.
pub fn is_cancelled(&self) -> bool {
self.0.load(Ordering::SeqCst)
}
/// `Err(Cancelled)` once the flag is set.
///
/// # Errors
/// [`DeployError::Cancelled`].
pub fn check(&self) -> Result<()> {
if self.is_cancelled() {
Err(DeployError::Cancelled)
} else {
Ok(())
}
}
}
/// One entry of a remote folder.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Entry {
/// The entry's name, checked by [`safe_name`].
pub name: String,
/// Whether it is a folder.
pub is_dir: bool,
/// Its size in bytes, when the server said.
pub size: Option<u64>,
}
/// Whether a name from a remote listing may become a local path segment.
pub fn safe_name(name: &str) -> bool {
!name.is_empty()
&& name != "."
&& name != ".."
&& name.len() <= 255
&& !name.contains(['/', '\\'])
&& !name.chars().any(char::is_control)
}
/// Called after every chunk with its size; returning an error (such as
/// [`DeployError::Cancelled`]) stops the transfer.
pub type Tick<'a> = &'a mut dyn FnMut(u64) -> Result<()>;
/// A connected server.
pub trait Remote {
/// Creates `dir` and every missing parent.
///
/// # Errors
/// [`DeployError::Transfer`] when the server refuses.
fn mkdir_all(&mut self, dir: &str) -> Result<()>;
/// Writes `data` to the file `path`, replacing it.
///
/// # Errors
/// [`DeployError::Transfer`], or what `tick` returned.
fn put(&mut self, path: &str, data: &mut dyn Read, tick: Tick<'_>) -> Result<()>;
/// Reads the file `path` into `out`.
///
/// # Errors
/// [`DeployError::Transfer`], or what `tick` returned.
fn get(&mut self, path: &str, out: &mut dyn Write, tick: Tick<'_>) -> Result<()>;
/// Whether `path` is a folder (`Some(true)`), a file (`Some(false)`)
/// or missing (`None`).
///
/// # Errors
/// [`DeployError::Transfer`] when the server fails otherwise.
fn kind(&mut self, path: &str) -> Result<Option<bool>>;
/// The entries of the folder `path`, without unsafe names, at most
/// [`MAX_LISTING`]. Meant for downloads now and the remote browser
/// later.
///
/// # Errors
/// [`DeployError::Transfer`] when the server refuses.
fn list(&mut self, path: &str) -> Result<Vec<Entry>>;
/// Ends the session politely; failures are ignored because the
/// connection is dropped anyway.
fn close(&mut self);
}
/// Copies `from` into `to` in chunks, calling `tick` after each.
///
/// # Errors
/// [`DeployError::Transfer`] on a read or write failure, or what `tick`
/// returned.
pub fn copy_chunks(from: &mut dyn Read, to: &mut dyn Write, tick: Tick<'_>) -> Result<u64> {
let mut buf = vec![0u8; CHUNK];
let mut total = 0u64;
loop {
let n = from
.read(&mut buf)
.map_err(|e| DeployError::Transfer(format!("read failed: {e}")))?;
if n == 0 {
break;
}
let chunk = buf.get(..n).unwrap_or_default();
to.write_all(chunk)
.map_err(|e| DeployError::Transfer(format!("write failed: {e}")))?;
total = total.saturating_add(n as u64);
tick(n as u64)?;
}
to.flush()
.map_err(|e| DeployError::Transfer(format!("write failed: {e}")))?;
Ok(total)
}
/// A local or mounted folder as a server.
#[derive(Debug)]
pub struct Folder {
/// The deployment root.
root: PathBuf,
}
impl Folder {
/// Opens the folder `root`.
///
/// # Errors
/// [`DeployError::Connect`] when it is not an existing folder.
pub fn open(root: &Path) -> Result<Self> {
if !root.is_dir() {
return Err(DeployError::Connect {
host: root.display().to_string(),
message: "the folder does not exist or is not mounted".into(),
});
}
Ok(Self {
root: root.to_path_buf(),
})
}
/// The local path of the remote `path`, always inside the root.
fn resolve(&self, path: &str) -> Result<PathBuf> {
let rel = normalize_relative(path).ok_or_else(|| {
DeployError::Invalid(format!("{path:?} leaves the deployment folder"))
})?;
Ok(if rel.is_empty() {
self.root.clone()
} else {
self.root.join(rel)
})
}
}
impl Remote for Folder {
fn mkdir_all(&mut self, dir: &str) -> Result<()> {
let path = self.resolve(dir)?;
std::fs::create_dir_all(&path).map_err(|e| io_error(&path, &e))
}
/// Writes a temporary file next to the target and renames it. A failed
/// or cancelled write removes the temporary file; if even that fails it
/// is left, and the next upload of the same file replaces it.
fn put(&mut self, path: &str, data: &mut dyn Read, tick: Tick<'_>) -> Result<()> {
let target = self.resolve(path)?;
let temp = target.with_extension("lutra-upload");
let result = (|| {
let mut file = std::fs::File::create(&temp).map_err(|e| io_error(&temp, &e))?;
copy_chunks(data, &mut file, tick)?;
drop(file);
std::fs::rename(&temp, &target).map_err(|e| io_error(&target, &e))
})();
if result.is_err() {
let _ = std::fs::remove_file(&temp);
}
result
}
fn get(&mut self, path: &str, out: &mut dyn Write, tick: Tick<'_>) -> Result<()> {
let source = self.resolve(path)?;
let mut file = std::fs::File::open(&source).map_err(|e| io_error(&source, &e))?;
copy_chunks(&mut file, out, tick).map(|_| ())
}
fn kind(&mut self, path: &str) -> Result<Option<bool>> {
let path = self.resolve(path)?;
match std::fs::metadata(&path) {
Ok(meta) => Ok(Some(meta.is_dir())),
Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(None),
Err(e) => Err(io_error(&path, &e)),
}
}
fn list(&mut self, path: &str) -> Result<Vec<Entry>> {
let dir = self.resolve(path)?;
let mut entries = Vec::new();
for entry in std::fs::read_dir(&dir).map_err(|e| io_error(&dir, &e))? {
let entry = entry.map_err(|e| io_error(&dir, &e))?;
let name = entry.file_name().to_string_lossy().into_owned();
if !safe_name(&name) {
continue;
}
let Ok(meta) = entry.metadata() else {
continue;
};
entries.push(Entry {
name,
is_dir: meta.is_dir(),
size: meta.is_file().then_some(meta.len()),
});
if entries.len() >= MAX_LISTING {
break;
}
}
entries.sort_by(|a, b| a.name.cmp(&b.name));
Ok(entries)
}
fn close(&mut self) {}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn names() {
assert!(safe_name("a.php"));
for bad in ["", ".", "..", "a/b", "a\\b", "a\nb"] {
assert!(!safe_name(bad), "{bad:?}");
}
}
#[test]
fn folder_round_trip() {
let dir = tempfile::tempdir().unwrap();
let mut f = Folder::open(dir.path()).unwrap();
f.mkdir_all("/a/b").unwrap();
let mut ticks = 0u64;
f.put("a/b/x.txt", &mut &b"hello"[..], &mut |n| {
ticks += n;
Ok(())
})
.unwrap();
assert_eq!(ticks, 5);
assert_eq!(f.kind("a/b").unwrap(), Some(true));
assert_eq!(f.kind("a/b/x.txt").unwrap(), Some(false));
assert_eq!(f.kind("nope").unwrap(), None);
let mut out = Vec::new();
f.get("a/b/x.txt", &mut out, &mut |_| Ok(())).unwrap();
assert_eq!(out, b"hello");
assert_eq!(f.list("a/b").unwrap()[0].name, "x.txt");
assert!(f.resolve("../x").is_err());
let cancel = Cancel::new();
cancel.cancel();
let err = f
.put("a/y.txt", &mut &b"z"[..], &mut |_| cancel.check())
.unwrap_err();
assert_eq!(err, DeployError::Cancelled);
assert_eq!(f.kind("a/y.txt").unwrap(), None);
assert!(Folder::open(&dir.path().join("missing")).is_err());
}
}