D1: engine deploy.* commands, deployProgress/deployLog events, upload on save
Servers per project in the config dir (no secrets), secrets in the keychain via a swappable SecretStore, test connection with the host-key and missing-secret questions as values, trust host key, upload, download, cancel, log. One job at a time; uploads on save queue. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
0644b044f2
commit
f3c9ef9ce3
8 changed files with 1637 additions and 0 deletions
|
|
@ -12,6 +12,7 @@ publish.workspace = true
|
|||
layer = 4
|
||||
|
||||
[dependencies]
|
||||
lutra-deploy.workspace = true
|
||||
lutra-index.workspace = true
|
||||
lutra-lsp.workspace = true
|
||||
lutra-proc.workspace = true
|
||||
|
|
@ -29,6 +30,7 @@ thiserror = "2.0.21"
|
|||
workspace = true
|
||||
|
||||
[dev-dependencies]
|
||||
lutra-deploy = { workspace = true, features = ["testserver"] }
|
||||
lutra-testkit.workspace = true
|
||||
proptest.workspace = true
|
||||
tempfile.workspace = true
|
||||
|
|
|
|||
1227
crates/engine/src/cmd/deploy.rs
Normal file
1227
crates/engine/src/cmd/deploy.rs
Normal file
File diff suppressed because it is too large
Load diff
|
|
@ -103,10 +103,12 @@ fn save(state: &mut State, command: &str, id: BufferId) -> Result<u64, EngineErr
|
|||
let was_dirty = open.dirty();
|
||||
open.saved_version = open.version;
|
||||
let version = open.version;
|
||||
let path = open.path.clone();
|
||||
journal::forget(state, id);
|
||||
if was_dirty {
|
||||
state.emit(Event::DirtyChanged { id, dirty: false });
|
||||
}
|
||||
super::deploy::saved(state, &path.to_string_lossy().replace('\\', "/"));
|
||||
Ok(version)
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@
|
|||
|
||||
mod app;
|
||||
mod buffer;
|
||||
pub mod deploy;
|
||||
mod file;
|
||||
pub mod git;
|
||||
pub mod git_blame;
|
||||
|
|
@ -82,6 +83,7 @@ pub fn register_builtins(registry: &mut Registry) -> Result<(), EngineError> {
|
|||
git_stash::register(registry)?;
|
||||
git_chunks::register(registry)?;
|
||||
git_ignore::register(registry)?;
|
||||
deploy::register(registry)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
|
|
@ -92,6 +94,7 @@ pub fn register_builtins(registry: &mut Registry) -> Result<(), EngineError> {
|
|||
pub(crate) const BUILTIN_COMMAND_IDS: &[&[&str]] = &[
|
||||
app::COMMAND_IDS,
|
||||
buffer::COMMAND_IDS,
|
||||
deploy::COMMAND_IDS,
|
||||
file::COMMAND_IDS,
|
||||
git::COMMAND_IDS,
|
||||
git_blame::COMMAND_IDS,
|
||||
|
|
|
|||
|
|
@ -154,6 +154,7 @@ impl Engine {
|
|||
crate::lsp::pump(state);
|
||||
cmd::git::pump(state);
|
||||
cmd::git_remote::pump(state);
|
||||
cmd::deploy::pump(state);
|
||||
Ok(())
|
||||
})?;
|
||||
Ok(state.take_events())
|
||||
|
|
@ -268,6 +269,13 @@ impl Engine {
|
|||
self.state.lsp.set_php_lint(php);
|
||||
}
|
||||
|
||||
/// Keeps deployment passwords and passphrases in `store` instead of
|
||||
/// the OS keychain: for tests, and for hosts that must not touch the
|
||||
/// keychain.
|
||||
pub fn set_secret_store(&mut self, store: std::sync::Arc<dyn lutra_deploy::SecretStore>) {
|
||||
self.state.deploy.set_secret_store(store);
|
||||
}
|
||||
|
||||
/// Read access to the state.
|
||||
pub fn state(&self) -> &State {
|
||||
&self.state
|
||||
|
|
|
|||
|
|
@ -209,6 +209,13 @@ pub enum Event {
|
|||
LspProgress(crate::lsp::LspProgress),
|
||||
/// The running fetch, pull, push or clone: `{active, operation?, phase?, percent?}`; `active: false` once it ended (M4.10).
|
||||
GitProgress(crate::cmd::git_remote::GitProgress),
|
||||
/// The running deployment job: `{active, operation?, server?, phase, file?, filesDone, filesTotal, bytesDone, bytesTotal}`; `active: false` once it ended (G17).
|
||||
DeployProgress(crate::cmd::deploy::DeployProgress),
|
||||
/// A new line of the deployment log: `{entry: {id, time, level, server, message}}` (G17).
|
||||
DeployLog {
|
||||
/// The line.
|
||||
entry: crate::cmd::deploy::LogEntry,
|
||||
},
|
||||
/// Terminal events (`terminalOutput`, `terminalExited`), see [`crate::cmd::terminal`].
|
||||
#[serde(untagged)]
|
||||
Terminal(crate::cmd::terminal::TerminalEvent),
|
||||
|
|
|
|||
|
|
@ -53,6 +53,8 @@ pub struct State {
|
|||
pub git_lines: crate::cmd::git_lines::GitLines,
|
||||
/// The running fetch, pull, push or clone (M4.10).
|
||||
pub git_remote: crate::cmd::git_remote::GitRemote,
|
||||
/// Deployment jobs, their log and the secret store (G17).
|
||||
pub deploy: crate::cmd::deploy::DeployState,
|
||||
/// Number of notifications emitted, for their ids.
|
||||
notifications: u64,
|
||||
/// Events queued for the host, oldest first.
|
||||
|
|
|
|||
386
crates/engine/tests/deploy.rs
Normal file
386
crates/engine/tests/deploy.rs
Normal file
|
|
@ -0,0 +1,386 @@
|
|||
//! The `deploy.*` commands through the engine (G17) against the local test
|
||||
//! servers of `lutra-deploy` (127.0.0.1 only, no real network), with an
|
||||
//! in-memory secret store instead of the OS keychain: servers and secrets,
|
||||
//! the host-key question, upload and download with `deployProgress` and
|
||||
//! `deployLog`, upload on save, cancel, one job at a time, bad params, and
|
||||
//! that the password never reaches a file, an event, a value or an error.
|
||||
|
||||
#![allow(
|
||||
clippy::unwrap_used,
|
||||
clippy::expect_used,
|
||||
clippy::panic,
|
||||
clippy::indexing_slicing
|
||||
)]
|
||||
|
||||
use std::path::Path;
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use lutra_deploy::MemoryStore;
|
||||
use lutra_deploy::testserver::{FtpServer, SftpServer};
|
||||
use lutra_engine::{Engine, EngineConfig, EngineError, Event, run_waiting};
|
||||
use serde_json::{Value, json};
|
||||
|
||||
const PASSWORD: &str = "Otter-Engine-S3cret#";
|
||||
|
||||
/// An engine behind a lock, like a host, recording every event and every
|
||||
/// value and error text.
|
||||
struct Host {
|
||||
engine: Mutex<Engine>,
|
||||
events: Mutex<Vec<Event>>,
|
||||
said: Mutex<Vec<String>>,
|
||||
}
|
||||
|
||||
impl Host {
|
||||
fn new(config: &Path) -> Arc<Self> {
|
||||
let mut engine = Engine::with_config(EngineConfig {
|
||||
config_dir: config.join("config"),
|
||||
data_dir: config.join("data"),
|
||||
})
|
||||
.unwrap();
|
||||
engine.set_secret_store(Arc::new(MemoryStore::default()));
|
||||
Arc::new(Self {
|
||||
engine: Mutex::new(engine),
|
||||
events: Mutex::new(Vec::new()),
|
||||
said: Mutex::new(Vec::new()),
|
||||
})
|
||||
}
|
||||
|
||||
fn run(&self, command: &str, params: Value) -> Result<Value, EngineError> {
|
||||
let result = run_waiting(
|
||||
|| self.engine.lock().unwrap(),
|
||||
command,
|
||||
params,
|
||||
|events| self.events.lock().unwrap().extend_from_slice(events),
|
||||
)
|
||||
.map(|o| o.value);
|
||||
let text = match &result {
|
||||
Ok(v) => v.to_string(),
|
||||
Err(e) => format!("{e} {e:?}"),
|
||||
};
|
||||
self.said.lock().unwrap().push(text);
|
||||
result
|
||||
}
|
||||
|
||||
fn pump(&self) {
|
||||
let events = self.engine.lock().unwrap().pump_vfs().unwrap();
|
||||
self.events.lock().unwrap().extend(events);
|
||||
}
|
||||
|
||||
fn seen(&self, kind: &str) -> Vec<Value> {
|
||||
self.events
|
||||
.lock()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.map(|e| serde_json::to_value(e).unwrap())
|
||||
.filter(|e| e["kind"] == kind)
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Everything the host saw: events, values, errors.
|
||||
fn transcript(&self) -> String {
|
||||
let mut text: Vec<String> = self
|
||||
.events
|
||||
.lock()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.map(|e| serde_json::to_string(e).unwrap())
|
||||
.collect();
|
||||
text.extend(self.said.lock().unwrap().iter().cloned());
|
||||
text.join("\n")
|
||||
}
|
||||
}
|
||||
|
||||
/// Every file under `dir`, read as text where possible.
|
||||
fn all_files(dir: &Path) -> String {
|
||||
let mut out = String::new();
|
||||
let Ok(entries) = std::fs::read_dir(dir) else {
|
||||
return out;
|
||||
};
|
||||
for entry in entries.flatten() {
|
||||
let path = entry.path();
|
||||
if path.is_dir() {
|
||||
out.push_str(&all_files(&path));
|
||||
} else {
|
||||
out.push_str(&String::from_utf8_lossy(&std::fs::read(&path).unwrap()));
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
fn project(root: &Path) -> std::path::PathBuf {
|
||||
let p = root.join("project");
|
||||
std::fs::create_dir_all(p.join("public")).unwrap();
|
||||
std::fs::write(p.join("public/index.php"), "<?php echo 1;").unwrap();
|
||||
std::fs::write(p.join("composer.json"), "{}").unwrap();
|
||||
p
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sftp_servers_host_key_upload_download_and_no_leak() {
|
||||
let tmp = tempfile::tempdir().unwrap();
|
||||
let remote = tmp.path().join("remote");
|
||||
std::fs::create_dir_all(&remote).unwrap();
|
||||
let sftp = SftpServer::start(&remote, "deploy", Some(PASSWORD), None, 11).unwrap();
|
||||
let host = Host::new(tmp.path());
|
||||
let proj = project(tmp.path());
|
||||
host.run("project.open", json!({"path": proj})).unwrap();
|
||||
|
||||
assert_eq!(
|
||||
host.run("deploy.servers", Value::Null).unwrap()["servers"],
|
||||
json!([])
|
||||
);
|
||||
let saved = host
|
||||
.run(
|
||||
"deploy.saveServer",
|
||||
json!({"server": {
|
||||
"name": "Staging", "kind": "sftp", "host": "127.0.0.1", "port": sftp.addr.port(),
|
||||
"user": "deploy", "root": "/www", "mappings": [{"local": "public", "remote": "html"}],
|
||||
}, "secret": PASSWORD}),
|
||||
)
|
||||
.unwrap();
|
||||
let id = saved["server"]["id"].as_str().unwrap().to_string();
|
||||
assert_eq!(saved["server"]["hasSecret"], true);
|
||||
assert_eq!(saved["server"]["secretKind"], "password");
|
||||
let listed = host.run("deploy.servers", Value::Null).unwrap();
|
||||
assert_eq!(listed["defaultServer"], json!(id));
|
||||
|
||||
let first = host.run("deploy.testConnection", json!({})).unwrap();
|
||||
assert_eq!(first["ok"], false);
|
||||
assert_eq!(first["hostKey"]["status"], "unknown");
|
||||
assert_eq!(first["hostKey"]["fingerprint"], json!(sftp.fingerprint));
|
||||
host.run(
|
||||
"deploy.trustHostKey",
|
||||
json!({"id": id, "algorithm": first["hostKey"]["algorithm"], "fingerprint": first["hostKey"]["fingerprint"]}),
|
||||
)
|
||||
.unwrap();
|
||||
let second = host
|
||||
.run("deploy.testConnection", json!({"id": id}))
|
||||
.unwrap();
|
||||
assert_eq!(second["ok"], true, "{second}");
|
||||
|
||||
let up = host.run("deploy.upload", json!({"paths": [""]})).unwrap();
|
||||
assert_eq!(up["files"], 1, "{up}");
|
||||
assert_eq!(up["skipped"], 1, "composer.json is not mapped: {up}");
|
||||
assert_eq!(
|
||||
std::fs::read_to_string(remote.join("www/html/index.php")).unwrap(),
|
||||
"<?php echo 1;"
|
||||
);
|
||||
let progress = host.seen("deployProgress");
|
||||
assert_eq!(progress.last().unwrap()["active"], false);
|
||||
assert_eq!(progress.last().unwrap()["operation"], "upload");
|
||||
assert_eq!(progress.last().unwrap()["filesDone"], 1);
|
||||
assert!(host.seen("deployLog").iter().any(|e| {
|
||||
e["entry"]["message"]
|
||||
.as_str()
|
||||
.unwrap()
|
||||
.contains("public/index.php")
|
||||
}));
|
||||
|
||||
std::fs::write(remote.join("www/html/index.php"), "<?php echo 2;").unwrap();
|
||||
let down = host
|
||||
.run("deploy.download", json!({"paths": ["public/index.php"]}))
|
||||
.unwrap();
|
||||
assert_eq!(down["files"], 1);
|
||||
assert_eq!(
|
||||
std::fs::read_to_string(proj.join("public/index.php")).unwrap(),
|
||||
"<?php echo 2;"
|
||||
);
|
||||
|
||||
let log = host.run("deploy.log", Value::Null).unwrap();
|
||||
assert!(log["entries"].as_array().unwrap().len() >= 3);
|
||||
host.run("deploy.clearLog", Value::Null).unwrap();
|
||||
assert_eq!(
|
||||
host.run("deploy.log", Value::Null).unwrap()["entries"],
|
||||
json!([])
|
||||
);
|
||||
|
||||
let wrong = host
|
||||
.run("deploy.setSecret", json!({"id": id, "secret": "nope"}))
|
||||
.unwrap();
|
||||
assert_eq!(wrong["stored"], true);
|
||||
let refused = host.run("deploy.testConnection", json!({})).unwrap();
|
||||
assert_eq!(refused["authFailed"], true, "{refused}");
|
||||
host.run("deploy.clearSecret", json!({"id": id})).unwrap();
|
||||
let missing = host.run("deploy.testConnection", json!({})).unwrap();
|
||||
assert_eq!(missing["needsSecret"], "password", "{missing}");
|
||||
assert!(
|
||||
host.run("deploy.upload", json!({"paths": ["public"]}))
|
||||
.is_err()
|
||||
);
|
||||
|
||||
assert!(
|
||||
host.run("deploy.removeServer", json!({"id": id})).unwrap()["removed"]
|
||||
.as_bool()
|
||||
.unwrap()
|
||||
);
|
||||
|
||||
host.pump();
|
||||
let transcript = host.transcript();
|
||||
assert!(
|
||||
!transcript.contains(PASSWORD),
|
||||
"the password leaked into events, values or errors"
|
||||
);
|
||||
let files = all_files(&tmp.path().join("config")) + &all_files(&tmp.path().join("data"));
|
||||
assert!(
|
||||
files.contains("known_hosts") || files.contains("SHA256:"),
|
||||
"the config dir holds the deployment files"
|
||||
);
|
||||
assert!(
|
||||
!files.contains(PASSWORD),
|
||||
"the password leaked into the config or data directory"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ftp_upload_on_save() {
|
||||
let tmp = tempfile::tempdir().unwrap();
|
||||
let remote = tmp.path().join("remote");
|
||||
std::fs::create_dir_all(&remote).unwrap();
|
||||
let ftp = FtpServer::start(&remote, "deploy", PASSWORD).unwrap();
|
||||
let host = Host::new(tmp.path());
|
||||
let proj = project(tmp.path());
|
||||
host.run("project.open", json!({"path": proj})).unwrap();
|
||||
let saved = host
|
||||
.run(
|
||||
"deploy.saveServer",
|
||||
json!({"server": {
|
||||
"name": "Shared host", "kind": "ftp", "host": "127.0.0.1", "port": ftp.addr.port(),
|
||||
"user": "deploy", "root": "/", "uploadOnSave": true, "excluded": ["composer.json"],
|
||||
}, "secret": PASSWORD}),
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(saved["server"]["uploadOnSave"], true);
|
||||
|
||||
let opened = host
|
||||
.run("file.open", json!({"path": "public/index.php"}))
|
||||
.unwrap();
|
||||
let id = opened["id"].as_u64().unwrap();
|
||||
host.run(
|
||||
"buffer.edit",
|
||||
json!({"id": id, "baseVersion": 1, "edits": [{"from": 0, "to": 0, "insert": "// saved\n"}]}),
|
||||
)
|
||||
.unwrap();
|
||||
host.run("file.save", json!({"id": id})).unwrap();
|
||||
let deadline = Instant::now() + Duration::from_secs(20);
|
||||
let uploaded = remote.join("public/index.php");
|
||||
while Instant::now() < deadline
|
||||
&& !host
|
||||
.seen("deployProgress")
|
||||
.iter()
|
||||
.any(|p| p["active"] == false)
|
||||
{
|
||||
host.pump();
|
||||
std::thread::sleep(Duration::from_millis(20));
|
||||
}
|
||||
assert_eq!(
|
||||
std::fs::read_to_string(&uploaded).unwrap(),
|
||||
"// saved\n<?php echo 1;"
|
||||
);
|
||||
|
||||
assert!(!host.transcript().contains(PASSWORD));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sftp_cancel_and_one_job_at_a_time() {
|
||||
let tmp = tempfile::tempdir().unwrap();
|
||||
let remote = tmp.path().join("remote");
|
||||
std::fs::create_dir_all(&remote).unwrap();
|
||||
let sftp = SftpServer::start(&remote, "deploy", Some(PASSWORD), None, 12).unwrap();
|
||||
let host = Host::new(tmp.path());
|
||||
let proj = project(tmp.path());
|
||||
std::fs::write(proj.join("big.bin"), vec![1u8; 48 * 1024 * 1024]).unwrap();
|
||||
host.run("project.open", json!({"path": proj})).unwrap();
|
||||
let saved = host
|
||||
.run(
|
||||
"deploy.saveServer",
|
||||
json!({"server": {
|
||||
"name": "Big", "kind": "sftp", "host": "127.0.0.1", "port": sftp.addr.port(),
|
||||
"user": "deploy", "root": "",
|
||||
}, "secret": PASSWORD}),
|
||||
)
|
||||
.unwrap();
|
||||
let id = saved["server"]["id"].as_str().unwrap().to_string();
|
||||
host.run(
|
||||
"deploy.trustHostKey",
|
||||
json!({"id": id, "algorithm": sftp.algorithm, "fingerprint": sftp.fingerprint}),
|
||||
)
|
||||
.unwrap();
|
||||
let engine_host = Arc::clone(&host);
|
||||
let worker =
|
||||
std::thread::spawn(move || engine_host.run("deploy.upload", json!({"paths": ["big.bin"]})));
|
||||
let deadline = Instant::now() + Duration::from_secs(30);
|
||||
let mut cancelled = false;
|
||||
while Instant::now() < deadline && !cancelled {
|
||||
host.pump();
|
||||
let running = host
|
||||
.seen("deployProgress")
|
||||
.iter()
|
||||
.any(|p| p["active"] == true && p["bytesDone"].as_u64().unwrap_or(0) > 0);
|
||||
if running {
|
||||
let second = host.run("deploy.testConnection", json!({}));
|
||||
assert!(
|
||||
second.is_err(),
|
||||
"a second job must be refused while one runs"
|
||||
);
|
||||
assert_eq!(
|
||||
host.run("deploy.cancel", Value::Null).unwrap()["cancelled"],
|
||||
true
|
||||
);
|
||||
cancelled = true;
|
||||
}
|
||||
std::thread::sleep(Duration::from_millis(5));
|
||||
}
|
||||
assert!(cancelled, "the upload never reported progress");
|
||||
let err = worker.join().unwrap().unwrap_err();
|
||||
assert!(err.to_string().contains("cancelled"), "{err}");
|
||||
assert!(!host.transcript().contains(PASSWORD));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bad_params_and_no_project() {
|
||||
let tmp = tempfile::tempdir().unwrap();
|
||||
let host = Host::new(tmp.path());
|
||||
assert!(matches!(
|
||||
host.run("deploy.servers", Value::Null),
|
||||
Err(EngineError::Disabled(_))
|
||||
));
|
||||
let proj = project(tmp.path());
|
||||
host.run("project.open", json!({"path": proj})).unwrap();
|
||||
let bad = [
|
||||
(
|
||||
"deploy.saveServer",
|
||||
json!({"server": {"name": "x", "kind": "sftp", "host": "-x", "user": "u"}}),
|
||||
),
|
||||
(
|
||||
"deploy.saveServer",
|
||||
json!({"server": {"name": "x", "kind": "ftp", "host": "h", "user": "u", "password": "p"}}),
|
||||
),
|
||||
(
|
||||
"deploy.saveServer",
|
||||
json!({"server": {"name": "x", "kind": "gopher", "host": "h", "user": "u"}}),
|
||||
),
|
||||
("deploy.setDefault", json!({"id": "missing"})),
|
||||
("deploy.upload", json!({"paths": []})),
|
||||
("deploy.upload", json!({"paths": ["a"], "id": "missing"})),
|
||||
(
|
||||
"deploy.trustHostKey",
|
||||
json!({"id": "missing", "algorithm": "a", "fingerprint": "b"}),
|
||||
),
|
||||
];
|
||||
for (command, params) in bad {
|
||||
let err = host.run(command, params.clone()).unwrap_err();
|
||||
assert_eq!(err.code(), "invalid_params", "{command} {params}: {err}");
|
||||
}
|
||||
let none = host
|
||||
.run("deploy.upload", json!({"paths": ["public"]}))
|
||||
.unwrap_err();
|
||||
assert!(
|
||||
none.to_string().contains("no default deployment server"),
|
||||
"{none}"
|
||||
);
|
||||
assert!(matches!(
|
||||
host.run("deploy.cancel", Value::Null),
|
||||
Err(EngineError::Disabled(_))
|
||||
));
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue