From f3c9ef9ce3ec3f428c40f48e55b167d33c63589b Mon Sep 17 00:00:00 2001 From: danny Date: Mon, 2 Nov 2026 19:23:36 +0100 Subject: [PATCH] D1: engine deploy.* commands, deployProgress/deployLog events, upload on save Servers per project in the config dir (no secrets), secrets in the keychain via a swappable SecretStore, test connection with the host-key and missing-secret questions as values, trust host key, upload, download, cancel, log. One job at a time; uploads on save queue. Co-Authored-By: Claude Opus 5.5 --- crates/engine/Cargo.toml | 2 + crates/engine/src/cmd/deploy.rs | 1227 +++++++++++++++++++++++++++++++ crates/engine/src/cmd/file.rs | 2 + crates/engine/src/cmd/mod.rs | 3 + crates/engine/src/engine.rs | 8 + crates/engine/src/event.rs | 7 + crates/engine/src/state.rs | 2 + crates/engine/tests/deploy.rs | 386 ++++++++++ 8 files changed, 1637 insertions(+) create mode 100644 crates/engine/src/cmd/deploy.rs create mode 100644 crates/engine/tests/deploy.rs diff --git a/crates/engine/Cargo.toml b/crates/engine/Cargo.toml index 98f9fa0..759713f 100644 --- a/crates/engine/Cargo.toml +++ b/crates/engine/Cargo.toml @@ -12,6 +12,7 @@ publish.workspace = true layer = 4 [dependencies] +lutra-deploy.workspace = true lutra-index.workspace = true lutra-lsp.workspace = true lutra-proc.workspace = true @@ -29,6 +30,7 @@ thiserror = "2.0.21" workspace = true [dev-dependencies] +lutra-deploy = { workspace = true, features = ["testserver"] } lutra-testkit.workspace = true proptest.workspace = true tempfile.workspace = true diff --git a/crates/engine/src/cmd/deploy.rs b/crates/engine/src/cmd/deploy.rs new file mode 100644 index 0000000..0058ff2 --- /dev/null +++ b/crates/engine/src/cmd/deploy.rs @@ -0,0 +1,1227 @@ +//! Deployment commands (G17, owner decision D27): servers per project, +//! secrets in the OS keychain, test connection, host keys, upload, +//! download, upload on save, cancel and the deployment log, through +//! `lutra-deploy`. +//! +//! **Storage.** A project's servers live in +//! `/deployment/.json` (the project id is the +//! journal's FNV hash of the root), so nothing is written into the project. +//! Trusted SFTP host keys live in `/deployment/known_hosts`. +//! Passwords and passphrases live only in the keychain under the account +//! `//`; no value, event, log +//! line or error ever contains one. +//! +//! **One job at a time.** Test connection, upload and download share one +//! slot; a second is refused while one runs, except uploads on save, which +//! wait in a queue and start when the slot is free. A job runs on a +//! `lutra-deploy` thread. Commands return a pending outcome; uploads on +//! save run in the background and are finished by [`pump`]. +//! +//! **Events.** `deployProgress {active, operation, server, phase, file?, +//! filesDone, filesTotal, bytesDone, bytesTotal}` from the pump when it +//! changed, `active: false` when the job ends; `deployLog {entry}` for +//! every new log line (`{id, time, level, server, message}`, `time` in +//! milliseconds since the epoch). The log keeps the newest 1,000 lines. + +use std::collections::VecDeque; +use std::path::PathBuf; +use std::sync::mpsc; +use std::sync::{Arc, Mutex, PoisonError}; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; + +use lutra_deploy::known_hosts::{self, HostKey}; +use lutra_deploy::secrets::account; +use lutra_deploy::{ + Cancel, DeployError, Deployment, Job, Keychain, Level, Monitor, Progress, Secret, SecretStore, + Server, job, +}; +use serde::{Deserialize, Serialize}; +use serde_json::{Value, json}; + +use super::params::parse; +use crate::journal::hex_hash; +use crate::{ + Command, CommandSpec, EngineError, Event, NotificationLevel, Outcome, Pending, Registry, State, +}; + +/// Longest wait for a job: transfers can take long; the transports time +/// out on a stalled connection first. +const WAIT: Duration = Duration::from_secs(6 * 60 * 60); + +/// Most lines kept in the deployment log. +const MAX_LOG: usize = 1000; + +/// Most paths in one upload or download request. +const MAX_PATHS: usize = 10_000; + +/// Most uploads on save waiting for the slot. +const MAX_QUEUE: usize = 1000; + +/// Registers the `deploy.*` commands. +/// +/// # Errors +/// [`EngineError::Internal`] if an id is taken. +pub fn register(registry: &mut Registry) -> Result<(), EngineError> { + registry.register(Box::new(Servers))?; + registry.register(Box::new(SaveServer))?; + registry.register(Box::new(RemoveServer))?; + registry.register(Box::new(SetDefault))?; + registry.register(Box::new(SetSecret))?; + registry.register(Box::new(ClearSecret))?; + registry.register(Box::new(TestConnection))?; + registry.register(Box::new(TrustHostKey))?; + registry.register(Box::new(Upload))?; + registry.register(Box::new(Download))?; + registry.register(Box::new(CancelJob))?; + registry.register(Box::new(Log))?; + registry.register(Box::new(ClearLog))?; + Ok(()) +} + +/// Ids of this module's commands. +#[cfg(test)] +pub(crate) const COMMAND_IDS: &[&str] = &[ + "deploy.cancel", + "deploy.clearLog", + "deploy.clearSecret", + "deploy.download", + "deploy.log", + "deploy.removeServer", + "deploy.saveServer", + "deploy.servers", + "deploy.setDefault", + "deploy.setSecret", + "deploy.testConnection", + "deploy.trustHostKey", + "deploy.upload", +]; + +/// Payload of [`Event::DeployProgress`]. +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct DeployProgress { + /// A job runs. + pub active: bool, + /// `test`, `upload` or `download`. + #[serde(skip_serializing_if = "Option::is_none")] + pub operation: Option, + /// The server's name. + #[serde(skip_serializing_if = "Option::is_none")] + pub server: Option, + /// The job's progress. + #[serde(flatten)] + pub progress: Progress, +} + +/// One line of the deployment log, as `deploy.log` and `deployLog` send it. +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct LogEntry { + /// Increasing id. + pub id: u64, + /// Milliseconds since the Unix epoch. + pub time: u64, + /// Severity. + pub level: Level, + /// The server's name. + pub server: String, + /// The text. + pub message: String, +} + +/// Where a background upload's thread leaves its result. +type BackgroundSlot = Arc>>>; + +/// The running job. +#[derive(Debug)] +struct Active { + /// `test`, `upload` or `download`. + operation: &'static str, + /// The server's name. + server: String, + /// Stops it. + cancel: Cancel, + /// Its progress and log. + monitor: Monitor, + /// Set for uploads on save: the pump finishes it. + background: Option, +} + +/// One upload on save waiting for the slot. +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] +struct Queued { + /// The server's id. + server: String, + /// The project-relative path. + path: String, +} + +/// State of the deployment commands. +#[derive(Debug)] +pub struct DeployState { + /// Where secrets are kept: the OS keychain unless a host or test chose + /// another store. + secrets: Arc, + /// The running job. + active: Option, + /// The `deployProgress` sent last. + sent: Option, + /// The newest log lines. + log: VecDeque, + /// The id of the next log line. + next_id: u64, + /// Uploads on save waiting for the slot, in order, without duplicates. + queue: VecDeque, +} + +impl Default for DeployState { + fn default() -> Self { + Self { + secrets: Arc::new(Keychain), + active: None, + sent: None, + log: VecDeque::new(), + next_id: 1, + queue: VecDeque::new(), + } + } +} + +impl DeployState { + /// Uses `store` for passwords and passphrases instead of the keychain. + pub fn set_secret_store(&mut self, store: Arc) { + self.secrets = store; + } +} + +/// Where a project's configuration and the known hosts live. +struct Paths { + /// The project root. + root: PathBuf, + /// The project id used in file and account names. + id: String, + /// The project's deployment configuration. + config: PathBuf, + /// Lutra's known-hosts file. + known_hosts: PathBuf, +} + +/// The open project's paths. +/// +/// # Errors +/// [`EngineError::Disabled`] without a project, [`EngineError::Failed`] +/// without a config directory. +fn paths(state: &State, command: &str) -> Result { + let root = state + .project + .as_ref() + .map(|w| w.project.root().to_path_buf()) + .ok_or_else(|| EngineError::Disabled(command.to_string()))?; + let dir = state + .config + .as_ref() + .map(|c| c.config_dir.join("deployment")) + .ok_or_else(|| EngineError::Failed("deployment needs a config directory".into()))?; + let id = hex_hash(root.to_string_lossy().as_bytes()); + Ok(Paths { + config: dir.join(format!("{id}.json")), + known_hosts: dir.join("known_hosts"), + root, + id, + }) +} + +/// A crate error as the engine's: the caller's mistakes are +/// `invalid_params`, everything else `failed`. +fn engine_error(command: &str, error: DeployError) -> EngineError { + if error.is_invalid_input() { + EngineError::invalid_params(command, error.to_string()) + } else { + EngineError::Failed(error.to_string()) + } +} + +/// Loads the open project's configuration. +fn load(state: &State, command: &str) -> Result<(Paths, Deployment), EngineError> { + let paths = paths(state, command)?; + let deployment = Deployment::load(&paths.config).map_err(|e| engine_error(command, e))?; + Ok((paths, deployment)) +} + +/// The server `id`, or the default one when `id` is `None`. +fn pick<'a>( + deployment: &'a Deployment, + id: Option<&str>, + command: &str, +) -> Result<&'a Server, EngineError> { + match id { + Some(id) => deployment + .server(id) + .ok_or_else(|| EngineError::invalid_params(command, format!("no server {id:?}"))), + None => deployment.default_target().ok_or_else(|| { + EngineError::Failed( + "no default deployment server: add one, or choose the default in the deployment settings".into(), + ) + }), + } +} + +/// The keychain secret of `server`, if it needs one and one is stored. +fn secret(state: &State, paths: &Paths, server: &Server) -> Result, EngineError> { + let Some(kind) = server.secret_kind() else { + return Ok(None); + }; + state + .deploy + .secrets + .get(&account(&paths.id, &server.id, kind)) + .map_err(|e| EngineError::Failed(e.to_string())) +} + +/// Milliseconds since the epoch. +fn now_ms() -> u64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|d| u64::try_from(d.as_millis()).unwrap_or(u64::MAX)) + .unwrap_or(0) +} + +/// Adds a log line and sends `deployLog`. +fn log(state: &mut State, level: Level, server: &str, message: String) { + let entry = LogEntry { + id: state.deploy.next_id, + time: now_ms(), + level, + server: server.to_string(), + message, + }; + state.deploy.next_id += 1; + state.deploy.log.push_back(entry.clone()); + while state.deploy.log.len() > MAX_LOG { + state.deploy.log.pop_front(); + } + state.emit(Event::DeployLog { entry }); +} + +/// Moves the running job's new log lines into the log. +fn drain(state: &mut State) { + let Some(active) = &state.deploy.active else { + return; + }; + let server = active.server.clone(); + for line in active.monitor.take_lines() { + log(state, line.level, &server, line.message); + } +} + +/// What a job does. +#[derive(Debug, Clone)] +enum Work { + /// Test the connection. + Test, + /// Upload these paths. + Upload(Vec), + /// Download these paths. + Download(Vec), +} + +impl Work { + /// The operation's name. + fn name(&self) -> &'static str { + match self { + Self::Test => "test", + Self::Upload(_) => "upload", + Self::Download(_) => "download", + } + } +} + +/// A job's result as the command's value. A missing secret or an unknown +/// or changed host key is a value with `ok: false` for test connection, so +/// the UI can ask; every other failure is an error. +fn test_value(result: Result) -> Result { + match result { + Ok(message) => Ok(json!({"ok": true, "message": message})), + Err(e @ DeployError::HostKeyUnknown { .. }) => { + let DeployError::HostKeyUnknown { + host, + algorithm, + fingerprint, + } = &e + else { + return Err(EngineError::Failed(e.to_string())); + }; + Ok(json!({"ok": false, "message": e.to_string(), "hostKey": { + "status": "unknown", "host": host, "algorithm": algorithm, "fingerprint": fingerprint, + }})) + } + Err(e @ DeployError::HostKeyMismatch { .. }) => { + let DeployError::HostKeyMismatch { + host, + algorithm, + fingerprint, + known, + } = &e + else { + return Err(EngineError::Failed(e.to_string())); + }; + Ok(json!({"ok": false, "message": e.to_string(), "hostKey": { + "status": "changed", "host": host, "algorithm": algorithm, "fingerprint": fingerprint, "known": known, + }})) + } + Err(e @ DeployError::MissingSecret { .. }) => { + let DeployError::MissingSecret { kind, .. } = &e else { + return Err(EngineError::Failed(e.to_string())); + }; + Ok(json!({"ok": false, "message": e.to_string(), "needsSecret": kind})) + } + Err(e @ DeployError::Auth(_)) => { + Ok(json!({"ok": false, "message": e.to_string(), "authFailed": true})) + } + Err(e) => Err(EngineError::Failed(e.to_string())), + } +} + +/// Builds the job for `server`. +fn make_job(state: &State, paths: &Paths, server: &Server) -> Result { + Ok(Job { + server: server.clone(), + project_root: paths.root.clone(), + secret: secret(state, paths, server)?, + known_hosts: paths.known_hosts.clone(), + cancel: Cancel::new(), + monitor: Monitor::new(), + }) +} + +/// Takes the slot for `job` and runs `run` on a new thread. `background` +/// is the result slot of an upload on save, which [`pump`] finishes. +fn spawn( + state: &mut State, + work: Work, + job: Job, + background: Option, + run: impl FnOnce(&Job, &Work) + Send + 'static, +) -> Result<(), EngineError> { + if let Some(active) = &state.deploy.active { + return Err(EngineError::Failed(format!( + "another deployment job is running ({} on {}); wait for it or cancel it", + active.operation, active.server + ))); + } + state.deploy.active = Some(Active { + operation: work.name(), + server: job.server.name.clone(), + cancel: job.cancel.clone(), + monitor: job.monitor.clone(), + background, + }); + state.deploy.sent = None; + let spawned = std::thread::Builder::new() + .name("lutra-deploy".into()) + .spawn(move || run(&job, &work)); + if let Err(e) = spawned { + state.deploy.active = None; + return Err(EngineError::Failed(format!( + "cannot start the deployment job: {e}" + ))); + } + Ok(()) +} + +/// Starts a job for a command and returns its pending outcome. The thread +/// ignores a failed wake-up send: it only means the outcome was dropped +/// because the host stopped waiting. +fn start( + state: &mut State, + command: &'static str, + server_id: Option<&str>, + work: Work, +) -> Result { + let (paths, deployment) = load(state, command)?; + let server = pick(&deployment, server_id, command)?.clone(); + let job = make_job(state, &paths, &server)?; + let slot: Arc>>> = Arc::new(Mutex::new(None)); + let (tx, rx) = mpsc::channel(); + let filled = Arc::clone(&slot); + spawn(state, work, job, None, move |job, work| { + let value = match work { + Work::Test => test_value(job::test_connection(job)), + Work::Upload(paths) => summary_value(job::upload(job, paths)), + Work::Download(paths) => summary_value(job::download(job, paths)), + }; + *filled.lock().unwrap_or_else(PoisonError::into_inner) = Some(value); + let _ = tx.send(Ok(Value::Null)); + })?; + Ok(Outcome::pending(Pending::new(rx, WAIT, move |state, _| { + let value = slot + .lock() + .unwrap_or_else(PoisonError::into_inner) + .take() + .ok_or_else(|| EngineError::Failed("the deployment job did not answer in time".into())); + let value = value.and_then(|v| v); + finish(state, value.as_ref().err().map(ToString::to_string)); + value + }))) +} + +/// An upload or download summary as the command's value. +fn summary_value(result: Result) -> Result { + result + .map_err(|e| EngineError::Failed(e.to_string())) + .and_then(|s| serde_json::to_value(s).map_err(|e| EngineError::Failed(e.to_string()))) +} + +/// Ends the running job: logs its last lines and `error`, frees the slot, +/// sends `deployProgress {active: false}` and starts the next queued +/// upload on save. +fn finish(state: &mut State, error: Option) { + drain(state); + let Some(active) = state.deploy.active.take() else { + return; + }; + if let Some(error) = error { + log( + state, + Level::Error, + &active.server, + format!("{} failed: {error}", active.operation), + ); + } else { + log( + state, + Level::Info, + &active.server, + format!("{} finished", active.operation), + ); + } + state.deploy.sent = None; + state.emit(Event::DeployProgress(DeployProgress { + active: false, + operation: Some(active.operation.to_string()), + server: Some(active.server), + progress: active.monitor.progress(), + })); + start_queued(state); +} + +/// Starts the queued uploads on save of one server when the slot is free. +fn start_queued(state: &mut State) { + if state.deploy.active.is_some() { + return; + } + let Some(first) = state.deploy.queue.front().cloned() else { + return; + }; + let mut paths = Vec::new(); + state.deploy.queue.retain(|q| { + if q.server == first.server { + paths.push(q.path.clone()); + false + } else { + true + } + }); + let started = (|| { + let (dirs, deployment) = load(state, "deploy.upload")?; + let Some(server) = deployment.server(&first.server).cloned() else { + return Ok(()); + }; + let job = make_job(state, &dirs, &server)?; + let slot = Arc::new(Mutex::new(None)); + let filled = Arc::clone(&slot); + spawn( + state, + Work::Upload(paths.clone()), + job, + Some(slot), + move |job, work| { + if let Work::Upload(paths) = work { + let result = job::upload(job, paths); + *filled.lock().unwrap_or_else(PoisonError::into_inner) = Some(result); + } + }, + ) + })(); + if let Err(e) = started { + log( + state, + Level::Error, + &first.server, + format!("upload on save failed: {e}"), + ); + state.notify( + NotificationLevel::Error, + Some("notification.deploy.uploadOnSaveFailed"), + Some(format!("Upload on save failed: {e}")), + None, + ); + } +} + +/// Queues an upload of the project-relative `path` to every server with +/// upload on save. Called after a buffer was written by `file.save` or +/// `file.saveAll`. A configuration that cannot be read is logged and +/// otherwise ignored, so saving never fails because of deployment. +pub fn saved(state: &mut State, path: &str) { + if state.project.is_none() || state.config.is_none() { + return; + } + let Ok((_, deployment)) = load(state, "file.save") else { + return; + }; + for server in deployment.servers.iter().filter(|s| s.upload_on_save) { + if lutra_deploy::mapping::remote_path(server, path).is_none() { + continue; + } + let queued = Queued { + server: server.id.clone(), + path: path.to_string(), + }; + if !state.deploy.queue.contains(&queued) && state.deploy.queue.len() < MAX_QUEUE { + state.deploy.queue.push_back(queued); + } + } + start_queued(state); +} + +/// Sends `deployProgress` and `deployLog` for the running job, and +/// finishes a background upload that ended. Called from +/// [`crate::Engine::pump_vfs`]. +pub fn pump(state: &mut State) { + drain(state); + let Some(active) = &state.deploy.active else { + start_queued(state); + return; + }; + if let Some(slot) = &active.background { + let done = slot.lock().unwrap_or_else(PoisonError::into_inner).take(); + if let Some(result) = done { + let server = active.server.clone(); + let error = match result { + Ok(summary) if summary.failed == 0 => None, + Ok(summary) => Some(format!( + "{} of the files failed; see the deployment log", + summary.failed + )), + Err(DeployError::Cancelled) => Some("cancelled".into()), + Err(e) => Some(e.to_string()), + }; + if let Some(error) = &error + && error != "cancelled" + { + let mut params = serde_json::Map::new(); + params.insert("server".into(), json!(server)); + params.insert("error".into(), json!(error)); + state.notify( + NotificationLevel::Error, + Some("notification.deploy.uploadOnSaveFailed"), + Some(format!("Upload on save to {server} failed: {error}")), + Some(params), + ); + } + finish(state, error); + return; + } + } + let Some(active) = &state.deploy.active else { + return; + }; + let now = DeployProgress { + active: true, + operation: Some(active.operation.to_string()), + server: Some(active.server.clone()), + progress: active.monitor.progress(), + }; + if state.deploy.sent.as_ref() != Some(&now) { + state.deploy.sent = Some(now.clone()); + state.emit(Event::DeployProgress(now)); + } +} + +/// One server in `deploy.servers`: its definition and whether its secret +/// is stored. +fn server_json(state: &State, paths: &Paths, server: &Server) -> Value { + let mut value = serde_json::to_value(server).unwrap_or(Value::Null); + let stored = server.secret_kind().map(|kind| { + state + .deploy + .secrets + .get(&account(&paths.id, &server.id, kind)) + .ok() + .flatten() + .is_some() + }); + if let Some(map) = value.as_object_mut() { + map.insert("secretKind".into(), json!(server.secret_kind())); + map.insert("hasSecret".into(), json!(stored.unwrap_or(false))); + } + value +} + +/// The whole configuration as `deploy.servers` returns it. +fn servers_value(state: &State, paths: &Paths, deployment: &Deployment) -> Value { + let servers: Vec = deployment + .servers + .iter() + .map(|s| server_json(state, paths, s)) + .collect(); + json!({ + "servers": servers, + "defaultServer": deployment.default_target().map(|s| s.id.clone()), + }) +} + +/// No params. +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +struct NoParams {} + +/// `deploy.servers`. +struct Servers; + +impl Command for Servers { + fn spec(&self) -> &CommandSpec { + &CommandSpec { + id: "deploy.servers", + label: "Deployment: List Servers", + menu_path: None, + default_shortcut: None, + params: "none; returns { servers: [{ id, name, kind: sftp|ftp|ftps|folder, host, port?, user, root, auth: password|key|agent, keyPath?, mappings: [{ local, remote }], excluded: [string], uploadOnSave, secretKind: password|passphrase|null, hasSecret }], defaultServer: id|null }; never returns a secret", + } + } + + fn enabled(&self, state: &State) -> bool { + state.has_project() + } + + fn run(&self, state: &mut State, params: Value) -> Result { + let _: NoParams = parse(self.spec().id, params)?; + let (paths, deployment) = load(state, self.spec().id)?; + Ok(Outcome::value(servers_value(state, &paths, &deployment))) + } +} + +/// Params of `deploy.saveServer`. +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +struct SaveParams { + /// The server; without an id it is added under a fresh one. + server: Value, + /// A new password or passphrase for the keychain. + #[serde(default)] + secret: Option, +} + +/// `deploy.saveServer`. +struct SaveServer; + +impl Command for SaveServer { + fn spec(&self) -> &CommandSpec { + &CommandSpec { + id: "deploy.saveServer", + label: "Deployment: Save Server", + menu_path: None, + default_shortcut: None, + params: "{ server: { id?, name, kind, host, port?, user, root, auth?, keyPath?, mappings?, excluded?, uploadOnSave? }, secret?: password or passphrase, stored only in the OS keychain }; adds the server (fresh id when absent) or replaces the one with its id; returns { server } as deploy.servers lists it", + } + } + + fn enabled(&self, state: &State) -> bool { + state.has_project() + } + + fn run(&self, state: &mut State, params: Value) -> Result { + let command = self.spec().id; + let p: SaveParams = parse(command, params)?; + let (paths, mut deployment) = load(state, command)?; + let mut raw = p.server; + if let Some(map) = raw.as_object_mut() + && !map.contains_key("id") + { + map.insert("id".into(), json!(deployment.fresh_id())); + } + let server: Server = serde_json::from_value(raw) + .map_err(|e| EngineError::invalid_params(command, e.to_string()))?; + let secret = p + .secret + .map(Secret::new) + .transpose() + .map_err(|e| engine_error(command, e))?; + deployment + .upsert(server.clone()) + .map_err(|e| engine_error(command, e))?; + let saved = deployment + .server(&server.id) + .cloned() + .ok_or_else(|| EngineError::Failed("the server was not saved".into()))?; + if let Some(secret) = &secret { + let kind = saved.secret_kind().ok_or_else(|| { + EngineError::invalid_params( + command, + "this server signs in without a password or passphrase", + ) + })?; + state + .deploy + .secrets + .set(&account(&paths.id, &saved.id, kind), secret) + .map_err(|e| EngineError::Failed(e.to_string()))?; + } + deployment + .save(&paths.config) + .map_err(|e| engine_error(command, e))?; + Ok(Outcome::value( + json!({"server": server_json(state, &paths, &saved)}), + )) + } +} + +/// Params naming one server. +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +struct IdParams { + /// The server's id. + id: String, +} + +/// `deploy.removeServer`. +struct RemoveServer; + +impl Command for RemoveServer { + fn spec(&self) -> &CommandSpec { + &CommandSpec { + id: "deploy.removeServer", + label: "Deployment: Remove Server", + menu_path: None, + default_shortcut: None, + params: "{ id }; removes the server and its keychain entries; returns { removed: boolean }", + } + } + + fn enabled(&self, state: &State) -> bool { + state.has_project() + } + + /// Removes the keychain entries too; one that cannot be removed is + /// left there (it holds no reference to anything any more) and the + /// server is still removed. + fn run(&self, state: &mut State, params: Value) -> Result { + let command = self.spec().id; + let p: IdParams = parse(command, params)?; + let (paths, mut deployment) = load(state, command)?; + let removed = deployment.remove(&p.id); + if removed { + deployment + .save(&paths.config) + .map_err(|e| engine_error(command, e))?; + for kind in ["password", "passphrase"] { + let _ = state + .deploy + .secrets + .delete(&account(&paths.id, &p.id, kind)); + } + } + Ok(Outcome::value(json!({"removed": removed}))) + } +} + +/// Params of `deploy.setDefault`. +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +struct DefaultParams { + /// The server's id; `null` clears the choice. + id: Option, +} + +/// `deploy.setDefault`. +struct SetDefault; + +impl Command for SetDefault { + fn spec(&self) -> &CommandSpec { + &CommandSpec { + id: "deploy.setDefault", + label: "Deployment: Set Default Server", + menu_path: None, + default_shortcut: None, + params: "{ id: server id or null }; returns { defaultServer }", + } + } + + fn enabled(&self, state: &State) -> bool { + state.has_project() + } + + fn run(&self, state: &mut State, params: Value) -> Result { + let command = self.spec().id; + let p: DefaultParams = parse(command, params)?; + let (paths, mut deployment) = load(state, command)?; + if let Some(id) = &p.id + && deployment.server(id).is_none() + { + return Err(EngineError::invalid_params( + command, + format!("no server {id:?}"), + )); + } + deployment.default_server = p.id; + deployment + .save(&paths.config) + .map_err(|e| engine_error(command, e))?; + Ok(Outcome::value( + json!({"defaultServer": deployment.default_target().map(|s| s.id.clone())}), + )) + } +} + +/// Params of `deploy.setSecret`. +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +struct SecretParams { + /// The server's id. + id: String, + /// The password or passphrase. + secret: String, +} + +/// `deploy.setSecret`. +struct SetSecret; + +impl Command for SetSecret { + fn spec(&self) -> &CommandSpec { + &CommandSpec { + id: "deploy.setSecret", + label: "Deployment: Store Password", + menu_path: None, + default_shortcut: None, + params: "{ id, secret }; stores the server's password or passphrase in the OS keychain only; returns { stored: true }", + } + } + + fn enabled(&self, state: &State) -> bool { + state.has_project() + } + + fn run(&self, state: &mut State, params: Value) -> Result { + let command = self.spec().id; + let p: SecretParams = parse(command, params)?; + let secret = Secret::new(p.secret).map_err(|e| engine_error(command, e))?; + let (paths, deployment) = load(state, command)?; + let server = pick(&deployment, Some(&p.id), command)?; + let kind = server.secret_kind().ok_or_else(|| { + EngineError::invalid_params( + command, + "this server signs in without a password or passphrase", + ) + })?; + state + .deploy + .secrets + .set(&account(&paths.id, &server.id, kind), &secret) + .map_err(|e| EngineError::Failed(e.to_string()))?; + Ok(Outcome::value(json!({"stored": true}))) + } +} + +/// `deploy.clearSecret`. +struct ClearSecret; + +impl Command for ClearSecret { + fn spec(&self) -> &CommandSpec { + &CommandSpec { + id: "deploy.clearSecret", + label: "Deployment: Forget Password", + menu_path: None, + default_shortcut: None, + params: "{ id }; removes the server's password and passphrase from the OS keychain; returns { cleared: true }", + } + } + + fn enabled(&self, state: &State) -> bool { + state.has_project() + } + + fn run(&self, state: &mut State, params: Value) -> Result { + let command = self.spec().id; + let p: IdParams = parse(command, params)?; + let paths = paths(state, command)?; + for kind in ["password", "passphrase"] { + state + .deploy + .secrets + .delete(&account(&paths.id, &p.id, kind)) + .map_err(|e| EngineError::Failed(e.to_string()))?; + } + Ok(Outcome::value(json!({"cleared": true}))) + } +} + +/// Params with an optional server. +#[derive(Debug, Deserialize, Default)] +#[serde(deny_unknown_fields)] +struct ServerParams { + /// The server's id; the default server when absent. + #[serde(default)] + id: Option, +} + +/// `deploy.testConnection`. +struct TestConnection; + +impl Command for TestConnection { + fn spec(&self) -> &CommandSpec { + &CommandSpec { + id: "deploy.testConnection", + label: "Deployment: Test Connection", + menu_path: Some("Tools/Deployment/Test Connection"), + default_shortcut: None, + params: "{ id?: server id (default server when absent) }; returns { ok: true, message } or { ok: false, message, hostKey?: { status: unknown|changed, host, algorithm, fingerprint, known? }, needsSecret?: password|passphrase, authFailed?: true }; other failures are errors", + } + } + + fn enabled(&self, state: &State) -> bool { + state.has_project() && state.deploy.active.is_none() + } + + fn run(&self, state: &mut State, params: Value) -> Result { + let p: ServerParams = parse(self.spec().id, params)?; + start(state, self.spec().id, p.id.as_deref(), Work::Test) + } +} + +/// Params of `deploy.trustHostKey`. +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +struct TrustParams { + /// The server's id. + id: String, + /// The key's algorithm, as test connection reported it. + algorithm: String, + /// The fingerprint the user checked. + fingerprint: String, + /// Replace a different trusted key. + #[serde(default)] + replace: bool, +} + +/// `deploy.trustHostKey`. +struct TrustHostKey; + +impl Command for TrustHostKey { + fn spec(&self) -> &CommandSpec { + &CommandSpec { + id: "deploy.trustHostKey", + label: "Deployment: Trust Host Key", + menu_path: None, + default_shortcut: None, + params: "{ id, algorithm, fingerprint, replace?: boolean (replace a changed key) }; stores the key in Lutra's known-hosts file; returns { trusted: true }", + } + } + + fn enabled(&self, state: &State) -> bool { + state.has_project() + } + + fn run(&self, state: &mut State, params: Value) -> Result { + let command = self.spec().id; + let p: TrustParams = parse(command, params)?; + let (paths, deployment) = load(state, command)?; + let server = pick(&deployment, Some(&p.id), command)?; + let host = server.address(); + if p.replace { + known_hosts::forget(&paths.known_hosts, &host).map_err(|e| engine_error(command, e))?; + } + known_hosts::trust( + &paths.known_hosts, + &HostKey { + host, + algorithm: p.algorithm.clone(), + fingerprint: p.fingerprint.clone(), + }, + ) + .map_err(|e| engine_error(command, e))?; + let name = server.name.clone(); + log( + state, + Level::Info, + &name, + format!("trusted the host key {} {}", p.algorithm, p.fingerprint), + ); + Ok(Outcome::value(json!({"trusted": true}))) + } +} + +/// Params of `deploy.upload` and `deploy.download`. +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +struct TransferParams { + /// Project-relative (or absolute, inside the project) files or + /// folders; `""` is the whole project. + paths: Vec, + /// The server's id; the default server when absent. + #[serde(default)] + id: Option, +} + +/// Turns client paths into project-relative ones. +fn relative_paths( + state: &State, + command: &str, + given: &[String], +) -> Result, EngineError> { + if given.is_empty() || given.len() > MAX_PATHS { + return Err(EngineError::invalid_params( + command, + format!("paths needs 1 to {MAX_PATHS} entries"), + )); + } + given + .iter() + .map(|p| { + let rel = if p.is_empty() { + PathBuf::new() + } else { + super::params::project_path(state, command, p)? + }; + Ok(rel.to_string_lossy().replace('\\', "/")) + }) + .collect() +} + +/// `deploy.upload`. +struct Upload; + +impl Command for Upload { + fn spec(&self) -> &CommandSpec { + &CommandSpec { + id: "deploy.upload", + label: "Deployment: Upload to Server", + menu_path: Some("Tools/Deployment/Upload to Default Server"), + default_shortcut: None, + params: "{ paths: [project-relative file or folder, \"\" for the whole project], id?: server id (default server when absent) }; returns { files, bytes, skipped, failed }; progress as deployProgress, lines as deployLog; deploy.cancel stops it", + } + } + + fn enabled(&self, state: &State) -> bool { + state.has_project() && state.deploy.active.is_none() + } + + fn run(&self, state: &mut State, params: Value) -> Result { + let command = self.spec().id; + let p: TransferParams = parse(command, params)?; + let paths = relative_paths(state, command, &p.paths)?; + start(state, command, p.id.as_deref(), Work::Upload(paths)) + } +} + +/// `deploy.download`. +struct Download; + +impl Command for Download { + fn spec(&self) -> &CommandSpec { + &CommandSpec { + id: "deploy.download", + label: "Deployment: Download from Server", + menu_path: Some("Tools/Deployment/Download from Default Server"), + default_shortcut: None, + params: "{ paths: [project-relative file or folder], id?: server id (default server when absent) }; replaces the local files; returns { files, bytes, skipped, failed }; progress as deployProgress; deploy.cancel stops it", + } + } + + fn enabled(&self, state: &State) -> bool { + state.has_project() && state.deploy.active.is_none() + } + + fn run(&self, state: &mut State, params: Value) -> Result { + let command = self.spec().id; + let p: TransferParams = parse(command, params)?; + let paths = relative_paths(state, command, &p.paths)?; + start(state, command, p.id.as_deref(), Work::Download(paths)) + } +} + +/// `deploy.cancel`. +struct CancelJob; + +impl Command for CancelJob { + fn spec(&self) -> &CommandSpec { + &CommandSpec { + id: "deploy.cancel", + label: "Deployment: Cancel", + menu_path: None, + default_shortcut: None, + params: "none; stops the running test, upload or download (it then fails with \"cancelled\") and drops queued uploads on save; returns { cancelled: boolean }; disabled while none runs", + } + } + + fn enabled(&self, state: &State) -> bool { + state.deploy.active.is_some() + } + + fn run(&self, state: &mut State, params: Value) -> Result { + let _: NoParams = parse(self.spec().id, params)?; + state.deploy.queue.clear(); + let cancelled = state.deploy.active.as_ref().is_some_and(|active| { + active.cancel.cancel(); + true + }); + Ok(Outcome::value(json!({"cancelled": cancelled}))) + } +} + +/// `deploy.log`. +struct Log; + +impl Command for Log { + fn spec(&self) -> &CommandSpec { + &CommandSpec { + id: "deploy.log", + label: "Deployment: Log", + menu_path: None, + default_shortcut: None, + params: "none; returns { entries: [{ id, time (ms since epoch), level: info|warning|error, server, message }], active: deployProgress or null } oldest first, at most 1000", + } + } + + fn enabled(&self, _state: &State) -> bool { + true + } + + fn run(&self, state: &mut State, params: Value) -> Result { + let _: NoParams = parse(self.spec().id, params)?; + let entries: Vec<&LogEntry> = state.deploy.log.iter().collect(); + let active = state.deploy.active.as_ref().map(|a| DeployProgress { + active: true, + operation: Some(a.operation.to_string()), + server: Some(a.server.clone()), + progress: a.monitor.progress(), + }); + Ok(Outcome::value( + json!({"entries": entries, "active": active}), + )) + } +} + +/// `deploy.clearLog`. +struct ClearLog; + +impl Command for ClearLog { + fn spec(&self) -> &CommandSpec { + &CommandSpec { + id: "deploy.clearLog", + label: "Deployment: Clear Log", + menu_path: None, + default_shortcut: None, + params: "none; returns {}", + } + } + + fn enabled(&self, _state: &State) -> bool { + true + } + + fn run(&self, state: &mut State, params: Value) -> Result { + let _: NoParams = parse(self.spec().id, params)?; + state.deploy.log.clear(); + Ok(Outcome::value(json!({}))) + } +} diff --git a/crates/engine/src/cmd/file.rs b/crates/engine/src/cmd/file.rs index fc35781..d93a635 100644 --- a/crates/engine/src/cmd/file.rs +++ b/crates/engine/src/cmd/file.rs @@ -103,10 +103,12 @@ fn save(state: &mut State, command: &str, id: BufferId) -> Result Result<(), EngineError> { git_stash::register(registry)?; git_chunks::register(registry)?; git_ignore::register(registry)?; + deploy::register(registry)?; Ok(()) } @@ -92,6 +94,7 @@ pub fn register_builtins(registry: &mut Registry) -> Result<(), EngineError> { pub(crate) const BUILTIN_COMMAND_IDS: &[&[&str]] = &[ app::COMMAND_IDS, buffer::COMMAND_IDS, + deploy::COMMAND_IDS, file::COMMAND_IDS, git::COMMAND_IDS, git_blame::COMMAND_IDS, diff --git a/crates/engine/src/engine.rs b/crates/engine/src/engine.rs index d6b6cb9..b09b9f7 100644 --- a/crates/engine/src/engine.rs +++ b/crates/engine/src/engine.rs @@ -154,6 +154,7 @@ impl Engine { crate::lsp::pump(state); cmd::git::pump(state); cmd::git_remote::pump(state); + cmd::deploy::pump(state); Ok(()) })?; Ok(state.take_events()) @@ -268,6 +269,13 @@ impl Engine { self.state.lsp.set_php_lint(php); } + /// Keeps deployment passwords and passphrases in `store` instead of + /// the OS keychain: for tests, and for hosts that must not touch the + /// keychain. + pub fn set_secret_store(&mut self, store: std::sync::Arc) { + self.state.deploy.set_secret_store(store); + } + /// Read access to the state. pub fn state(&self) -> &State { &self.state diff --git a/crates/engine/src/event.rs b/crates/engine/src/event.rs index 804be6c..f7a6901 100644 --- a/crates/engine/src/event.rs +++ b/crates/engine/src/event.rs @@ -209,6 +209,13 @@ pub enum Event { LspProgress(crate::lsp::LspProgress), /// The running fetch, pull, push or clone: `{active, operation?, phase?, percent?}`; `active: false` once it ended (M4.10). GitProgress(crate::cmd::git_remote::GitProgress), + /// The running deployment job: `{active, operation?, server?, phase, file?, filesDone, filesTotal, bytesDone, bytesTotal}`; `active: false` once it ended (G17). + DeployProgress(crate::cmd::deploy::DeployProgress), + /// A new line of the deployment log: `{entry: {id, time, level, server, message}}` (G17). + DeployLog { + /// The line. + entry: crate::cmd::deploy::LogEntry, + }, /// Terminal events (`terminalOutput`, `terminalExited`), see [`crate::cmd::terminal`]. #[serde(untagged)] Terminal(crate::cmd::terminal::TerminalEvent), diff --git a/crates/engine/src/state.rs b/crates/engine/src/state.rs index f79b2b8..e7eb628 100644 --- a/crates/engine/src/state.rs +++ b/crates/engine/src/state.rs @@ -53,6 +53,8 @@ pub struct State { pub git_lines: crate::cmd::git_lines::GitLines, /// The running fetch, pull, push or clone (M4.10). pub git_remote: crate::cmd::git_remote::GitRemote, + /// Deployment jobs, their log and the secret store (G17). + pub deploy: crate::cmd::deploy::DeployState, /// Number of notifications emitted, for their ids. notifications: u64, /// Events queued for the host, oldest first. diff --git a/crates/engine/tests/deploy.rs b/crates/engine/tests/deploy.rs new file mode 100644 index 0000000..0a57f57 --- /dev/null +++ b/crates/engine/tests/deploy.rs @@ -0,0 +1,386 @@ +//! The `deploy.*` commands through the engine (G17) against the local test +//! servers of `lutra-deploy` (127.0.0.1 only, no real network), with an +//! in-memory secret store instead of the OS keychain: servers and secrets, +//! the host-key question, upload and download with `deployProgress` and +//! `deployLog`, upload on save, cancel, one job at a time, bad params, and +//! that the password never reaches a file, an event, a value or an error. + +#![allow( + clippy::unwrap_used, + clippy::expect_used, + clippy::panic, + clippy::indexing_slicing +)] + +use std::path::Path; +use std::sync::{Arc, Mutex}; +use std::time::{Duration, Instant}; + +use lutra_deploy::MemoryStore; +use lutra_deploy::testserver::{FtpServer, SftpServer}; +use lutra_engine::{Engine, EngineConfig, EngineError, Event, run_waiting}; +use serde_json::{Value, json}; + +const PASSWORD: &str = "Otter-Engine-S3cret#"; + +/// An engine behind a lock, like a host, recording every event and every +/// value and error text. +struct Host { + engine: Mutex, + events: Mutex>, + said: Mutex>, +} + +impl Host { + fn new(config: &Path) -> Arc { + let mut engine = Engine::with_config(EngineConfig { + config_dir: config.join("config"), + data_dir: config.join("data"), + }) + .unwrap(); + engine.set_secret_store(Arc::new(MemoryStore::default())); + Arc::new(Self { + engine: Mutex::new(engine), + events: Mutex::new(Vec::new()), + said: Mutex::new(Vec::new()), + }) + } + + fn run(&self, command: &str, params: Value) -> Result { + let result = run_waiting( + || self.engine.lock().unwrap(), + command, + params, + |events| self.events.lock().unwrap().extend_from_slice(events), + ) + .map(|o| o.value); + let text = match &result { + Ok(v) => v.to_string(), + Err(e) => format!("{e} {e:?}"), + }; + self.said.lock().unwrap().push(text); + result + } + + fn pump(&self) { + let events = self.engine.lock().unwrap().pump_vfs().unwrap(); + self.events.lock().unwrap().extend(events); + } + + fn seen(&self, kind: &str) -> Vec { + self.events + .lock() + .unwrap() + .iter() + .map(|e| serde_json::to_value(e).unwrap()) + .filter(|e| e["kind"] == kind) + .collect() + } + + /// Everything the host saw: events, values, errors. + fn transcript(&self) -> String { + let mut text: Vec = self + .events + .lock() + .unwrap() + .iter() + .map(|e| serde_json::to_string(e).unwrap()) + .collect(); + text.extend(self.said.lock().unwrap().iter().cloned()); + text.join("\n") + } +} + +/// Every file under `dir`, read as text where possible. +fn all_files(dir: &Path) -> String { + let mut out = String::new(); + let Ok(entries) = std::fs::read_dir(dir) else { + return out; + }; + for entry in entries.flatten() { + let path = entry.path(); + if path.is_dir() { + out.push_str(&all_files(&path)); + } else { + out.push_str(&String::from_utf8_lossy(&std::fs::read(&path).unwrap())); + } + } + out +} + +fn project(root: &Path) -> std::path::PathBuf { + let p = root.join("project"); + std::fs::create_dir_all(p.join("public")).unwrap(); + std::fs::write(p.join("public/index.php"), "= 3); + host.run("deploy.clearLog", Value::Null).unwrap(); + assert_eq!( + host.run("deploy.log", Value::Null).unwrap()["entries"], + json!([]) + ); + + let wrong = host + .run("deploy.setSecret", json!({"id": id, "secret": "nope"})) + .unwrap(); + assert_eq!(wrong["stored"], true); + let refused = host.run("deploy.testConnection", json!({})).unwrap(); + assert_eq!(refused["authFailed"], true, "{refused}"); + host.run("deploy.clearSecret", json!({"id": id})).unwrap(); + let missing = host.run("deploy.testConnection", json!({})).unwrap(); + assert_eq!(missing["needsSecret"], "password", "{missing}"); + assert!( + host.run("deploy.upload", json!({"paths": ["public"]})) + .is_err() + ); + + assert!( + host.run("deploy.removeServer", json!({"id": id})).unwrap()["removed"] + .as_bool() + .unwrap() + ); + + host.pump(); + let transcript = host.transcript(); + assert!( + !transcript.contains(PASSWORD), + "the password leaked into events, values or errors" + ); + let files = all_files(&tmp.path().join("config")) + &all_files(&tmp.path().join("data")); + assert!( + files.contains("known_hosts") || files.contains("SHA256:"), + "the config dir holds the deployment files" + ); + assert!( + !files.contains(PASSWORD), + "the password leaked into the config or data directory" + ); +} + +#[test] +fn ftp_upload_on_save() { + let tmp = tempfile::tempdir().unwrap(); + let remote = tmp.path().join("remote"); + std::fs::create_dir_all(&remote).unwrap(); + let ftp = FtpServer::start(&remote, "deploy", PASSWORD).unwrap(); + let host = Host::new(tmp.path()); + let proj = project(tmp.path()); + host.run("project.open", json!({"path": proj})).unwrap(); + let saved = host + .run( + "deploy.saveServer", + json!({"server": { + "name": "Shared host", "kind": "ftp", "host": "127.0.0.1", "port": ftp.addr.port(), + "user": "deploy", "root": "/", "uploadOnSave": true, "excluded": ["composer.json"], + }, "secret": PASSWORD}), + ) + .unwrap(); + assert_eq!(saved["server"]["uploadOnSave"], true); + + let opened = host + .run("file.open", json!({"path": "public/index.php"})) + .unwrap(); + let id = opened["id"].as_u64().unwrap(); + host.run( + "buffer.edit", + json!({"id": id, "baseVersion": 1, "edits": [{"from": 0, "to": 0, "insert": "// saved\n"}]}), + ) + .unwrap(); + host.run("file.save", json!({"id": id})).unwrap(); + let deadline = Instant::now() + Duration::from_secs(20); + let uploaded = remote.join("public/index.php"); + while Instant::now() < deadline + && !host + .seen("deployProgress") + .iter() + .any(|p| p["active"] == false) + { + host.pump(); + std::thread::sleep(Duration::from_millis(20)); + } + assert_eq!( + std::fs::read_to_string(&uploaded).unwrap(), + "// saved\n 0); + if running { + let second = host.run("deploy.testConnection", json!({})); + assert!( + second.is_err(), + "a second job must be refused while one runs" + ); + assert_eq!( + host.run("deploy.cancel", Value::Null).unwrap()["cancelled"], + true + ); + cancelled = true; + } + std::thread::sleep(Duration::from_millis(5)); + } + assert!(cancelled, "the upload never reported progress"); + let err = worker.join().unwrap().unwrap_err(); + assert!(err.to_string().contains("cancelled"), "{err}"); + assert!(!host.transcript().contains(PASSWORD)); +} + +#[test] +fn bad_params_and_no_project() { + let tmp = tempfile::tempdir().unwrap(); + let host = Host::new(tmp.path()); + assert!(matches!( + host.run("deploy.servers", Value::Null), + Err(EngineError::Disabled(_)) + )); + let proj = project(tmp.path()); + host.run("project.open", json!({"path": proj})).unwrap(); + let bad = [ + ( + "deploy.saveServer", + json!({"server": {"name": "x", "kind": "sftp", "host": "-x", "user": "u"}}), + ), + ( + "deploy.saveServer", + json!({"server": {"name": "x", "kind": "ftp", "host": "h", "user": "u", "password": "p"}}), + ), + ( + "deploy.saveServer", + json!({"server": {"name": "x", "kind": "gopher", "host": "h", "user": "u"}}), + ), + ("deploy.setDefault", json!({"id": "missing"})), + ("deploy.upload", json!({"paths": []})), + ("deploy.upload", json!({"paths": ["a"], "id": "missing"})), + ( + "deploy.trustHostKey", + json!({"id": "missing", "algorithm": "a", "fingerprint": "b"}), + ), + ]; + for (command, params) in bad { + let err = host.run(command, params.clone()).unwrap_err(); + assert_eq!(err.code(), "invalid_params", "{command} {params}: {err}"); + } + let none = host + .run("deploy.upload", json!({"paths": ["public"]})) + .unwrap_err(); + assert!( + none.to_string().contains("no default deployment server"), + "{none}" + ); + assert!(matches!( + host.run("deploy.cancel", Value::Null), + Err(EngineError::Disabled(_)) + )); +}