D1: engine deploy.* commands, deployProgress/deployLog events, upload on save

Servers per project in the config dir (no secrets), secrets in the
keychain via a swappable SecretStore, test connection with the host-key
and missing-secret questions as values, trust host key, upload,
download, cancel, log. One job at a time; uploads on save queue.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
danny 2026-11-02 19:23:36 +01:00
parent 0644b044f2
commit f3c9ef9ce3
8 changed files with 1637 additions and 0 deletions

View file

@ -12,6 +12,7 @@ publish.workspace = true
layer = 4
[dependencies]
lutra-deploy.workspace = true
lutra-index.workspace = true
lutra-lsp.workspace = true
lutra-proc.workspace = true
@ -29,6 +30,7 @@ thiserror = "2.0.21"
workspace = true
[dev-dependencies]
lutra-deploy = { workspace = true, features = ["testserver"] }
lutra-testkit.workspace = true
proptest.workspace = true
tempfile.workspace = true

File diff suppressed because it is too large Load diff

View file

@ -103,10 +103,12 @@ fn save(state: &mut State, command: &str, id: BufferId) -> Result<u64, EngineErr
let was_dirty = open.dirty();
open.saved_version = open.version;
let version = open.version;
let path = open.path.clone();
journal::forget(state, id);
if was_dirty {
state.emit(Event::DirtyChanged { id, dirty: false });
}
super::deploy::saved(state, &path.to_string_lossy().replace('\\', "/"));
Ok(version)
}

View file

@ -2,6 +2,7 @@
mod app;
mod buffer;
pub mod deploy;
mod file;
pub mod git;
pub mod git_blame;
@ -82,6 +83,7 @@ pub fn register_builtins(registry: &mut Registry) -> Result<(), EngineError> {
git_stash::register(registry)?;
git_chunks::register(registry)?;
git_ignore::register(registry)?;
deploy::register(registry)?;
Ok(())
}
@ -92,6 +94,7 @@ pub fn register_builtins(registry: &mut Registry) -> Result<(), EngineError> {
pub(crate) const BUILTIN_COMMAND_IDS: &[&[&str]] = &[
app::COMMAND_IDS,
buffer::COMMAND_IDS,
deploy::COMMAND_IDS,
file::COMMAND_IDS,
git::COMMAND_IDS,
git_blame::COMMAND_IDS,

View file

@ -154,6 +154,7 @@ impl Engine {
crate::lsp::pump(state);
cmd::git::pump(state);
cmd::git_remote::pump(state);
cmd::deploy::pump(state);
Ok(())
})?;
Ok(state.take_events())
@ -268,6 +269,13 @@ impl Engine {
self.state.lsp.set_php_lint(php);
}
/// Keeps deployment passwords and passphrases in `store` instead of
/// the OS keychain: for tests, and for hosts that must not touch the
/// keychain.
pub fn set_secret_store(&mut self, store: std::sync::Arc<dyn lutra_deploy::SecretStore>) {
self.state.deploy.set_secret_store(store);
}
/// Read access to the state.
pub fn state(&self) -> &State {
&self.state

View file

@ -209,6 +209,13 @@ pub enum Event {
LspProgress(crate::lsp::LspProgress),
/// The running fetch, pull, push or clone: `{active, operation?, phase?, percent?}`; `active: false` once it ended (M4.10).
GitProgress(crate::cmd::git_remote::GitProgress),
/// The running deployment job: `{active, operation?, server?, phase, file?, filesDone, filesTotal, bytesDone, bytesTotal}`; `active: false` once it ended (G17).
DeployProgress(crate::cmd::deploy::DeployProgress),
/// A new line of the deployment log: `{entry: {id, time, level, server, message}}` (G17).
DeployLog {
/// The line.
entry: crate::cmd::deploy::LogEntry,
},
/// Terminal events (`terminalOutput`, `terminalExited`), see [`crate::cmd::terminal`].
#[serde(untagged)]
Terminal(crate::cmd::terminal::TerminalEvent),

View file

@ -53,6 +53,8 @@ pub struct State {
pub git_lines: crate::cmd::git_lines::GitLines,
/// The running fetch, pull, push or clone (M4.10).
pub git_remote: crate::cmd::git_remote::GitRemote,
/// Deployment jobs, their log and the secret store (G17).
pub deploy: crate::cmd::deploy::DeployState,
/// Number of notifications emitted, for their ids.
notifications: u64,
/// Events queued for the host, oldest first.

View file

@ -0,0 +1,386 @@
//! The `deploy.*` commands through the engine (G17) against the local test
//! servers of `lutra-deploy` (127.0.0.1 only, no real network), with an
//! in-memory secret store instead of the OS keychain: servers and secrets,
//! the host-key question, upload and download with `deployProgress` and
//! `deployLog`, upload on save, cancel, one job at a time, bad params, and
//! that the password never reaches a file, an event, a value or an error.
#![allow(
clippy::unwrap_used,
clippy::expect_used,
clippy::panic,
clippy::indexing_slicing
)]
use std::path::Path;
use std::sync::{Arc, Mutex};
use std::time::{Duration, Instant};
use lutra_deploy::MemoryStore;
use lutra_deploy::testserver::{FtpServer, SftpServer};
use lutra_engine::{Engine, EngineConfig, EngineError, Event, run_waiting};
use serde_json::{Value, json};
const PASSWORD: &str = "Otter-Engine-S3cret#";
/// An engine behind a lock, like a host, recording every event and every
/// value and error text.
struct Host {
engine: Mutex<Engine>,
events: Mutex<Vec<Event>>,
said: Mutex<Vec<String>>,
}
impl Host {
fn new(config: &Path) -> Arc<Self> {
let mut engine = Engine::with_config(EngineConfig {
config_dir: config.join("config"),
data_dir: config.join("data"),
})
.unwrap();
engine.set_secret_store(Arc::new(MemoryStore::default()));
Arc::new(Self {
engine: Mutex::new(engine),
events: Mutex::new(Vec::new()),
said: Mutex::new(Vec::new()),
})
}
fn run(&self, command: &str, params: Value) -> Result<Value, EngineError> {
let result = run_waiting(
|| self.engine.lock().unwrap(),
command,
params,
|events| self.events.lock().unwrap().extend_from_slice(events),
)
.map(|o| o.value);
let text = match &result {
Ok(v) => v.to_string(),
Err(e) => format!("{e} {e:?}"),
};
self.said.lock().unwrap().push(text);
result
}
fn pump(&self) {
let events = self.engine.lock().unwrap().pump_vfs().unwrap();
self.events.lock().unwrap().extend(events);
}
fn seen(&self, kind: &str) -> Vec<Value> {
self.events
.lock()
.unwrap()
.iter()
.map(|e| serde_json::to_value(e).unwrap())
.filter(|e| e["kind"] == kind)
.collect()
}
/// Everything the host saw: events, values, errors.
fn transcript(&self) -> String {
let mut text: Vec<String> = self
.events
.lock()
.unwrap()
.iter()
.map(|e| serde_json::to_string(e).unwrap())
.collect();
text.extend(self.said.lock().unwrap().iter().cloned());
text.join("\n")
}
}
/// Every file under `dir`, read as text where possible.
fn all_files(dir: &Path) -> String {
let mut out = String::new();
let Ok(entries) = std::fs::read_dir(dir) else {
return out;
};
for entry in entries.flatten() {
let path = entry.path();
if path.is_dir() {
out.push_str(&all_files(&path));
} else {
out.push_str(&String::from_utf8_lossy(&std::fs::read(&path).unwrap()));
}
}
out
}
fn project(root: &Path) -> std::path::PathBuf {
let p = root.join("project");
std::fs::create_dir_all(p.join("public")).unwrap();
std::fs::write(p.join("public/index.php"), "<?php echo 1;").unwrap();
std::fs::write(p.join("composer.json"), "{}").unwrap();
p
}
#[test]
fn sftp_servers_host_key_upload_download_and_no_leak() {
let tmp = tempfile::tempdir().unwrap();
let remote = tmp.path().join("remote");
std::fs::create_dir_all(&remote).unwrap();
let sftp = SftpServer::start(&remote, "deploy", Some(PASSWORD), None, 11).unwrap();
let host = Host::new(tmp.path());
let proj = project(tmp.path());
host.run("project.open", json!({"path": proj})).unwrap();
assert_eq!(
host.run("deploy.servers", Value::Null).unwrap()["servers"],
json!([])
);
let saved = host
.run(
"deploy.saveServer",
json!({"server": {
"name": "Staging", "kind": "sftp", "host": "127.0.0.1", "port": sftp.addr.port(),
"user": "deploy", "root": "/www", "mappings": [{"local": "public", "remote": "html"}],
}, "secret": PASSWORD}),
)
.unwrap();
let id = saved["server"]["id"].as_str().unwrap().to_string();
assert_eq!(saved["server"]["hasSecret"], true);
assert_eq!(saved["server"]["secretKind"], "password");
let listed = host.run("deploy.servers", Value::Null).unwrap();
assert_eq!(listed["defaultServer"], json!(id));
let first = host.run("deploy.testConnection", json!({})).unwrap();
assert_eq!(first["ok"], false);
assert_eq!(first["hostKey"]["status"], "unknown");
assert_eq!(first["hostKey"]["fingerprint"], json!(sftp.fingerprint));
host.run(
"deploy.trustHostKey",
json!({"id": id, "algorithm": first["hostKey"]["algorithm"], "fingerprint": first["hostKey"]["fingerprint"]}),
)
.unwrap();
let second = host
.run("deploy.testConnection", json!({"id": id}))
.unwrap();
assert_eq!(second["ok"], true, "{second}");
let up = host.run("deploy.upload", json!({"paths": [""]})).unwrap();
assert_eq!(up["files"], 1, "{up}");
assert_eq!(up["skipped"], 1, "composer.json is not mapped: {up}");
assert_eq!(
std::fs::read_to_string(remote.join("www/html/index.php")).unwrap(),
"<?php echo 1;"
);
let progress = host.seen("deployProgress");
assert_eq!(progress.last().unwrap()["active"], false);
assert_eq!(progress.last().unwrap()["operation"], "upload");
assert_eq!(progress.last().unwrap()["filesDone"], 1);
assert!(host.seen("deployLog").iter().any(|e| {
e["entry"]["message"]
.as_str()
.unwrap()
.contains("public/index.php")
}));
std::fs::write(remote.join("www/html/index.php"), "<?php echo 2;").unwrap();
let down = host
.run("deploy.download", json!({"paths": ["public/index.php"]}))
.unwrap();
assert_eq!(down["files"], 1);
assert_eq!(
std::fs::read_to_string(proj.join("public/index.php")).unwrap(),
"<?php echo 2;"
);
let log = host.run("deploy.log", Value::Null).unwrap();
assert!(log["entries"].as_array().unwrap().len() >= 3);
host.run("deploy.clearLog", Value::Null).unwrap();
assert_eq!(
host.run("deploy.log", Value::Null).unwrap()["entries"],
json!([])
);
let wrong = host
.run("deploy.setSecret", json!({"id": id, "secret": "nope"}))
.unwrap();
assert_eq!(wrong["stored"], true);
let refused = host.run("deploy.testConnection", json!({})).unwrap();
assert_eq!(refused["authFailed"], true, "{refused}");
host.run("deploy.clearSecret", json!({"id": id})).unwrap();
let missing = host.run("deploy.testConnection", json!({})).unwrap();
assert_eq!(missing["needsSecret"], "password", "{missing}");
assert!(
host.run("deploy.upload", json!({"paths": ["public"]}))
.is_err()
);
assert!(
host.run("deploy.removeServer", json!({"id": id})).unwrap()["removed"]
.as_bool()
.unwrap()
);
host.pump();
let transcript = host.transcript();
assert!(
!transcript.contains(PASSWORD),
"the password leaked into events, values or errors"
);
let files = all_files(&tmp.path().join("config")) + &all_files(&tmp.path().join("data"));
assert!(
files.contains("known_hosts") || files.contains("SHA256:"),
"the config dir holds the deployment files"
);
assert!(
!files.contains(PASSWORD),
"the password leaked into the config or data directory"
);
}
#[test]
fn ftp_upload_on_save() {
let tmp = tempfile::tempdir().unwrap();
let remote = tmp.path().join("remote");
std::fs::create_dir_all(&remote).unwrap();
let ftp = FtpServer::start(&remote, "deploy", PASSWORD).unwrap();
let host = Host::new(tmp.path());
let proj = project(tmp.path());
host.run("project.open", json!({"path": proj})).unwrap();
let saved = host
.run(
"deploy.saveServer",
json!({"server": {
"name": "Shared host", "kind": "ftp", "host": "127.0.0.1", "port": ftp.addr.port(),
"user": "deploy", "root": "/", "uploadOnSave": true, "excluded": ["composer.json"],
}, "secret": PASSWORD}),
)
.unwrap();
assert_eq!(saved["server"]["uploadOnSave"], true);
let opened = host
.run("file.open", json!({"path": "public/index.php"}))
.unwrap();
let id = opened["id"].as_u64().unwrap();
host.run(
"buffer.edit",
json!({"id": id, "baseVersion": 1, "edits": [{"from": 0, "to": 0, "insert": "// saved\n"}]}),
)
.unwrap();
host.run("file.save", json!({"id": id})).unwrap();
let deadline = Instant::now() + Duration::from_secs(20);
let uploaded = remote.join("public/index.php");
while Instant::now() < deadline
&& !host
.seen("deployProgress")
.iter()
.any(|p| p["active"] == false)
{
host.pump();
std::thread::sleep(Duration::from_millis(20));
}
assert_eq!(
std::fs::read_to_string(&uploaded).unwrap(),
"// saved\n<?php echo 1;"
);
assert!(!host.transcript().contains(PASSWORD));
}
#[test]
fn sftp_cancel_and_one_job_at_a_time() {
let tmp = tempfile::tempdir().unwrap();
let remote = tmp.path().join("remote");
std::fs::create_dir_all(&remote).unwrap();
let sftp = SftpServer::start(&remote, "deploy", Some(PASSWORD), None, 12).unwrap();
let host = Host::new(tmp.path());
let proj = project(tmp.path());
std::fs::write(proj.join("big.bin"), vec![1u8; 48 * 1024 * 1024]).unwrap();
host.run("project.open", json!({"path": proj})).unwrap();
let saved = host
.run(
"deploy.saveServer",
json!({"server": {
"name": "Big", "kind": "sftp", "host": "127.0.0.1", "port": sftp.addr.port(),
"user": "deploy", "root": "",
}, "secret": PASSWORD}),
)
.unwrap();
let id = saved["server"]["id"].as_str().unwrap().to_string();
host.run(
"deploy.trustHostKey",
json!({"id": id, "algorithm": sftp.algorithm, "fingerprint": sftp.fingerprint}),
)
.unwrap();
let engine_host = Arc::clone(&host);
let worker =
std::thread::spawn(move || engine_host.run("deploy.upload", json!({"paths": ["big.bin"]})));
let deadline = Instant::now() + Duration::from_secs(30);
let mut cancelled = false;
while Instant::now() < deadline && !cancelled {
host.pump();
let running = host
.seen("deployProgress")
.iter()
.any(|p| p["active"] == true && p["bytesDone"].as_u64().unwrap_or(0) > 0);
if running {
let second = host.run("deploy.testConnection", json!({}));
assert!(
second.is_err(),
"a second job must be refused while one runs"
);
assert_eq!(
host.run("deploy.cancel", Value::Null).unwrap()["cancelled"],
true
);
cancelled = true;
}
std::thread::sleep(Duration::from_millis(5));
}
assert!(cancelled, "the upload never reported progress");
let err = worker.join().unwrap().unwrap_err();
assert!(err.to_string().contains("cancelled"), "{err}");
assert!(!host.transcript().contains(PASSWORD));
}
#[test]
fn bad_params_and_no_project() {
let tmp = tempfile::tempdir().unwrap();
let host = Host::new(tmp.path());
assert!(matches!(
host.run("deploy.servers", Value::Null),
Err(EngineError::Disabled(_))
));
let proj = project(tmp.path());
host.run("project.open", json!({"path": proj})).unwrap();
let bad = [
(
"deploy.saveServer",
json!({"server": {"name": "x", "kind": "sftp", "host": "-x", "user": "u"}}),
),
(
"deploy.saveServer",
json!({"server": {"name": "x", "kind": "ftp", "host": "h", "user": "u", "password": "p"}}),
),
(
"deploy.saveServer",
json!({"server": {"name": "x", "kind": "gopher", "host": "h", "user": "u"}}),
),
("deploy.setDefault", json!({"id": "missing"})),
("deploy.upload", json!({"paths": []})),
("deploy.upload", json!({"paths": ["a"], "id": "missing"})),
(
"deploy.trustHostKey",
json!({"id": "missing", "algorithm": "a", "fingerprint": "b"}),
),
];
for (command, params) in bad {
let err = host.run(command, params.clone()).unwrap_err();
assert_eq!(err.code(), "invalid_params", "{command} {params}: {err}");
}
let none = host
.run("deploy.upload", json!({"paths": ["public"]}))
.unwrap_err();
assert!(
none.to_string().contains("no default deployment server"),
"{none}"
);
assert!(matches!(
host.run("deploy.cancel", Value::Null),
Err(EngineError::Disabled(_))
));
}