D1: lutra-deploy crate: SFTP, FTP/FTPS and folder transports, keychain secrets, known hosts, path mappings
Pure Rust (D27): russh + russh-sftp with ring, suppaftp with rustls/ring, keyring (Windows Credential Manager, macOS Keychain). Lutra's own known-hosts file, refuse on mismatch. In-process SFTP and FTP test servers behind the testserver feature. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
33a7185d75
commit
0644b044f2
16 changed files with 5250 additions and 13 deletions
294
crates/deploy/src/remote.rs
Normal file
294
crates/deploy/src/remote.rs
Normal file
|
|
@ -0,0 +1,294 @@
|
|||
//! The operations every transport offers, cancellation and progress, and
|
||||
//! the folder transport.
|
||||
//!
|
||||
//! Remote paths are `/`-separated strings as [`crate::mapping`] builds
|
||||
//! them. Directory listings come from the server and are untrusted: names
|
||||
//! with `/`, `\`, control characters, `.` or `..` are dropped by
|
||||
//! [`safe_name`] before they become local paths.
|
||||
|
||||
use std::io::{Read, Write};
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
|
||||
use crate::config::io_error;
|
||||
use crate::error::{DeployError, Result};
|
||||
use crate::mapping::normalize_relative;
|
||||
|
||||
/// Size of one transfer chunk; progress and cancellation are checked
|
||||
/// between chunks.
|
||||
pub const CHUNK: usize = 32 * 1024;
|
||||
|
||||
/// Most entries read from one remote folder.
|
||||
pub const MAX_LISTING: usize = 100_000;
|
||||
|
||||
/// Stops a running job. Clones share the flag.
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub struct Cancel(Arc<AtomicBool>);
|
||||
|
||||
impl Cancel {
|
||||
/// A flag that is not set.
|
||||
pub fn new() -> Self {
|
||||
Self::default()
|
||||
}
|
||||
|
||||
/// Asks the job to stop.
|
||||
pub fn cancel(&self) {
|
||||
self.0.store(true, Ordering::SeqCst);
|
||||
}
|
||||
|
||||
/// Whether the job was asked to stop.
|
||||
pub fn is_cancelled(&self) -> bool {
|
||||
self.0.load(Ordering::SeqCst)
|
||||
}
|
||||
|
||||
/// `Err(Cancelled)` once the flag is set.
|
||||
///
|
||||
/// # Errors
|
||||
/// [`DeployError::Cancelled`].
|
||||
pub fn check(&self) -> Result<()> {
|
||||
if self.is_cancelled() {
|
||||
Err(DeployError::Cancelled)
|
||||
} else {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// One entry of a remote folder.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct Entry {
|
||||
/// The entry's name, checked by [`safe_name`].
|
||||
pub name: String,
|
||||
/// Whether it is a folder.
|
||||
pub is_dir: bool,
|
||||
/// Its size in bytes, when the server said.
|
||||
pub size: Option<u64>,
|
||||
}
|
||||
|
||||
/// Whether a name from a remote listing may become a local path segment.
|
||||
pub fn safe_name(name: &str) -> bool {
|
||||
!name.is_empty()
|
||||
&& name != "."
|
||||
&& name != ".."
|
||||
&& name.len() <= 255
|
||||
&& !name.contains(['/', '\\'])
|
||||
&& !name.chars().any(char::is_control)
|
||||
}
|
||||
|
||||
/// Called after every chunk with its size; returning an error (such as
|
||||
/// [`DeployError::Cancelled`]) stops the transfer.
|
||||
pub type Tick<'a> = &'a mut dyn FnMut(u64) -> Result<()>;
|
||||
|
||||
/// A connected server.
|
||||
pub trait Remote {
|
||||
/// Creates `dir` and every missing parent.
|
||||
///
|
||||
/// # Errors
|
||||
/// [`DeployError::Transfer`] when the server refuses.
|
||||
fn mkdir_all(&mut self, dir: &str) -> Result<()>;
|
||||
|
||||
/// Writes `data` to the file `path`, replacing it.
|
||||
///
|
||||
/// # Errors
|
||||
/// [`DeployError::Transfer`], or what `tick` returned.
|
||||
fn put(&mut self, path: &str, data: &mut dyn Read, tick: Tick<'_>) -> Result<()>;
|
||||
|
||||
/// Reads the file `path` into `out`.
|
||||
///
|
||||
/// # Errors
|
||||
/// [`DeployError::Transfer`], or what `tick` returned.
|
||||
fn get(&mut self, path: &str, out: &mut dyn Write, tick: Tick<'_>) -> Result<()>;
|
||||
|
||||
/// Whether `path` is a folder (`Some(true)`), a file (`Some(false)`)
|
||||
/// or missing (`None`).
|
||||
///
|
||||
/// # Errors
|
||||
/// [`DeployError::Transfer`] when the server fails otherwise.
|
||||
fn kind(&mut self, path: &str) -> Result<Option<bool>>;
|
||||
|
||||
/// The entries of the folder `path`, without unsafe names, at most
|
||||
/// [`MAX_LISTING`]. Meant for downloads now and the remote browser
|
||||
/// later.
|
||||
///
|
||||
/// # Errors
|
||||
/// [`DeployError::Transfer`] when the server refuses.
|
||||
fn list(&mut self, path: &str) -> Result<Vec<Entry>>;
|
||||
|
||||
/// Ends the session politely; failures are ignored because the
|
||||
/// connection is dropped anyway.
|
||||
fn close(&mut self);
|
||||
}
|
||||
|
||||
/// Copies `from` into `to` in chunks, calling `tick` after each.
|
||||
///
|
||||
/// # Errors
|
||||
/// [`DeployError::Transfer`] on a read or write failure, or what `tick`
|
||||
/// returned.
|
||||
pub fn copy_chunks(from: &mut dyn Read, to: &mut dyn Write, tick: Tick<'_>) -> Result<u64> {
|
||||
let mut buf = vec![0u8; CHUNK];
|
||||
let mut total = 0u64;
|
||||
loop {
|
||||
let n = from
|
||||
.read(&mut buf)
|
||||
.map_err(|e| DeployError::Transfer(format!("read failed: {e}")))?;
|
||||
if n == 0 {
|
||||
break;
|
||||
}
|
||||
let chunk = buf.get(..n).unwrap_or_default();
|
||||
to.write_all(chunk)
|
||||
.map_err(|e| DeployError::Transfer(format!("write failed: {e}")))?;
|
||||
total = total.saturating_add(n as u64);
|
||||
tick(n as u64)?;
|
||||
}
|
||||
to.flush()
|
||||
.map_err(|e| DeployError::Transfer(format!("write failed: {e}")))?;
|
||||
Ok(total)
|
||||
}
|
||||
|
||||
/// A local or mounted folder as a server.
|
||||
#[derive(Debug)]
|
||||
pub struct Folder {
|
||||
/// The deployment root.
|
||||
root: PathBuf,
|
||||
}
|
||||
|
||||
impl Folder {
|
||||
/// Opens the folder `root`.
|
||||
///
|
||||
/// # Errors
|
||||
/// [`DeployError::Connect`] when it is not an existing folder.
|
||||
pub fn open(root: &Path) -> Result<Self> {
|
||||
if !root.is_dir() {
|
||||
return Err(DeployError::Connect {
|
||||
host: root.display().to_string(),
|
||||
message: "the folder does not exist or is not mounted".into(),
|
||||
});
|
||||
}
|
||||
Ok(Self {
|
||||
root: root.to_path_buf(),
|
||||
})
|
||||
}
|
||||
|
||||
/// The local path of the remote `path`, always inside the root.
|
||||
fn resolve(&self, path: &str) -> Result<PathBuf> {
|
||||
let rel = normalize_relative(path).ok_or_else(|| {
|
||||
DeployError::Invalid(format!("{path:?} leaves the deployment folder"))
|
||||
})?;
|
||||
Ok(if rel.is_empty() {
|
||||
self.root.clone()
|
||||
} else {
|
||||
self.root.join(rel)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl Remote for Folder {
|
||||
fn mkdir_all(&mut self, dir: &str) -> Result<()> {
|
||||
let path = self.resolve(dir)?;
|
||||
std::fs::create_dir_all(&path).map_err(|e| io_error(&path, &e))
|
||||
}
|
||||
|
||||
/// Writes a temporary file next to the target and renames it. A failed
|
||||
/// or cancelled write removes the temporary file; if even that fails it
|
||||
/// is left, and the next upload of the same file replaces it.
|
||||
fn put(&mut self, path: &str, data: &mut dyn Read, tick: Tick<'_>) -> Result<()> {
|
||||
let target = self.resolve(path)?;
|
||||
let temp = target.with_extension("lutra-upload");
|
||||
let result = (|| {
|
||||
let mut file = std::fs::File::create(&temp).map_err(|e| io_error(&temp, &e))?;
|
||||
copy_chunks(data, &mut file, tick)?;
|
||||
drop(file);
|
||||
std::fs::rename(&temp, &target).map_err(|e| io_error(&target, &e))
|
||||
})();
|
||||
if result.is_err() {
|
||||
let _ = std::fs::remove_file(&temp);
|
||||
}
|
||||
result
|
||||
}
|
||||
|
||||
fn get(&mut self, path: &str, out: &mut dyn Write, tick: Tick<'_>) -> Result<()> {
|
||||
let source = self.resolve(path)?;
|
||||
let mut file = std::fs::File::open(&source).map_err(|e| io_error(&source, &e))?;
|
||||
copy_chunks(&mut file, out, tick).map(|_| ())
|
||||
}
|
||||
|
||||
fn kind(&mut self, path: &str) -> Result<Option<bool>> {
|
||||
let path = self.resolve(path)?;
|
||||
match std::fs::metadata(&path) {
|
||||
Ok(meta) => Ok(Some(meta.is_dir())),
|
||||
Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(None),
|
||||
Err(e) => Err(io_error(&path, &e)),
|
||||
}
|
||||
}
|
||||
|
||||
fn list(&mut self, path: &str) -> Result<Vec<Entry>> {
|
||||
let dir = self.resolve(path)?;
|
||||
let mut entries = Vec::new();
|
||||
for entry in std::fs::read_dir(&dir).map_err(|e| io_error(&dir, &e))? {
|
||||
let entry = entry.map_err(|e| io_error(&dir, &e))?;
|
||||
let name = entry.file_name().to_string_lossy().into_owned();
|
||||
if !safe_name(&name) {
|
||||
continue;
|
||||
}
|
||||
let Ok(meta) = entry.metadata() else {
|
||||
continue;
|
||||
};
|
||||
entries.push(Entry {
|
||||
name,
|
||||
is_dir: meta.is_dir(),
|
||||
size: meta.is_file().then_some(meta.len()),
|
||||
});
|
||||
if entries.len() >= MAX_LISTING {
|
||||
break;
|
||||
}
|
||||
}
|
||||
entries.sort_by(|a, b| a.name.cmp(&b.name));
|
||||
Ok(entries)
|
||||
}
|
||||
|
||||
fn close(&mut self) {}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn names() {
|
||||
assert!(safe_name("a.php"));
|
||||
for bad in ["", ".", "..", "a/b", "a\\b", "a\nb"] {
|
||||
assert!(!safe_name(bad), "{bad:?}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn folder_round_trip() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let mut f = Folder::open(dir.path()).unwrap();
|
||||
f.mkdir_all("/a/b").unwrap();
|
||||
let mut ticks = 0u64;
|
||||
f.put("a/b/x.txt", &mut &b"hello"[..], &mut |n| {
|
||||
ticks += n;
|
||||
Ok(())
|
||||
})
|
||||
.unwrap();
|
||||
assert_eq!(ticks, 5);
|
||||
assert_eq!(f.kind("a/b").unwrap(), Some(true));
|
||||
assert_eq!(f.kind("a/b/x.txt").unwrap(), Some(false));
|
||||
assert_eq!(f.kind("nope").unwrap(), None);
|
||||
let mut out = Vec::new();
|
||||
f.get("a/b/x.txt", &mut out, &mut |_| Ok(())).unwrap();
|
||||
assert_eq!(out, b"hello");
|
||||
assert_eq!(f.list("a/b").unwrap()[0].name, "x.txt");
|
||||
assert!(f.resolve("../x").is_err());
|
||||
let cancel = Cancel::new();
|
||||
cancel.cancel();
|
||||
let err = f
|
||||
.put("a/y.txt", &mut &b"z"[..], &mut |_| cancel.check())
|
||||
.unwrap_err();
|
||||
assert_eq!(err, DeployError::Cancelled);
|
||||
assert_eq!(f.kind("a/y.txt").unwrap(), None);
|
||||
assert!(Folder::open(&dir.path().join("missing")).is_err());
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue