diff --git a/Cargo.lock b/Cargo.lock index 35bb6ce..0a895b6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -407,6 +407,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6bb31b46c14244e20ee9984b11bf5c992b91fb6939fea616e3512c8baecdbe5f" dependencies = [ "memchr", + "regex-automata", "serde_core", ] @@ -1162,6 +1163,29 @@ version = "1.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4ef6b89e5b37196644d8796de5268852ff179b44e96276cf4290264843743bb7" +[[package]] +name = "encoding_rs" +version = "0.8.42" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e985e0451871ad22fb8d2b6b076e2028a502a0d3950998c2c5c0a4f9b5d9679" +dependencies = [ + "cfg-if", + "core_detect", + "multiversion_no_op", + "rustversion", + "scopeguard", + "simdutf8", +] + +[[package]] +name = "encoding_rs_io" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fba3fe847045ecff794b9c138293a80db914678c453ad63fbf0c6a9eb6e00b22" +dependencies = [ + "encoding_rs", +] + [[package]] name = "endi" version = "1.1.1" @@ -1705,6 +1729,43 @@ dependencies = [ "system-deps 6.2.2", ] +[[package]] +name = "grep-matcher" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9417543f4870fc8f1c8e1af870afae2431007626d9e703fce6471c468d33847" +dependencies = [ + "memchr", +] + +[[package]] +name = "grep-regex" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ce0c256c3ad82bcc07b812c15a45ec1d398122e8e15124f96695234db7112ef" +dependencies = [ + "bstr", + "grep-matcher", + "log", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "grep-searcher" +version = "0.1.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72348823a0eafc4bc2e9051064f28b5b42cc100b571b3a35d67918d711efcbc6" +dependencies = [ + "bstr", + "encoding_rs", + "encoding_rs_io", + "grep-matcher", + "log", + "memchr", + "memmap2", +] + [[package]] name = "gtk" version = "0.18.2" @@ -2521,6 +2582,8 @@ dependencies = [ name = "lutra-engine" version = "0.0.0" dependencies = [ + "lutra-search", + "lutra-testkit", "lutra-text", "lutra-vfs", "proptest", @@ -2545,6 +2608,17 @@ version = "0.0.0" [[package]] name = "lutra-search" version = "0.0.0" +dependencies = [ + "globset", + "grep-matcher", + "grep-regex", + "grep-searcher", + "ignore", + "lutra-testkit", + "proptest", + "tempfile", + "thiserror 2.0.21", +] [[package]] name = "lutra-syntax" @@ -2697,6 +2771,12 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "multiversion_no_op" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "743fb55ba31b18fb1ecef6bdc9aa2743314978ac084044301a7eee33fb99a20d" + [[package]] name = "ndk" version = "0.9.0" @@ -4072,6 +4152,12 @@ version = "0.3.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea" +[[package]] +name = "simdutf8" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" + [[package]] name = "siphasher" version = "1.0.4" diff --git a/crates/control/tests/search.rs b/crates/control/tests/search.rs new file mode 100644 index 0000000..b17f894 --- /dev/null +++ b/crates/control/tests/search.rs @@ -0,0 +1,170 @@ +//! Find in files over a real TCP control connection: `search.find` answers +//! at once, `searchResults` and `searchDone` arrive as event lines from the +//! pump, `search.cancel` ends a search. + +#![allow( + clippy::unwrap_used, + clippy::expect_used, + clippy::indexing_slicing, + clippy::panic +)] + +use std::io::{BufRead, BufReader, Write}; +use std::net::TcpStream; +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::time::Duration; + +use lutra_testkit::EngineBackend; +use serde_json::{Value, json}; + +/// A client that keeps responses and event lines apart. +struct Client { + reader: BufReader, + writer: TcpStream, + events: Vec, +} + +impl Client { + fn connect(addr: std::net::SocketAddr) -> Self { + let stream = TcpStream::connect(addr).unwrap(); + stream + .set_read_timeout(Some(Duration::from_secs(10))) + .unwrap(); + Self { + writer: stream.try_clone().unwrap(), + reader: BufReader::new(stream), + events: Vec::new(), + } + } + + fn line(&mut self) -> Value { + let mut line = String::new(); + self.reader.read_line(&mut line).expect("line within 10 s"); + serde_json::from_str(&line).unwrap() + } + + fn call(&mut self, request: Value) -> Value { + self.writer + .write_all(format!("{request}\n").as_bytes()) + .unwrap(); + loop { + let line = self.line(); + if line.get("event").is_some() { + self.events.push(line["event"].clone()); + } else { + return line; + } + } + } + + /// Reads event lines until `searchDone` for `id`; returns the + /// `searchResults` of that search and the done event. + fn until_done(&mut self, id: u64) -> (Vec, Value) { + let mut results = Vec::new(); + let mut pending = std::mem::take(&mut self.events).into_iter(); + loop { + let event = match pending.next() { + Some(e) => e, + None => { + let line = self.line(); + let Some(event) = line.get("event") else { + continue; + }; + event.clone() + } + }; + if event["searchId"] != id { + continue; + } + match event["kind"].as_str() { + Some("searchResults") => results.push(event), + Some("searchDone") => return (results, event), + _ => {} + } + } + } +} + +fn execute(command: &str, params: Value) -> Value { + json!({"id": command, "method": "engine.execute", "params": {"command": command, "params": params}}) +} + +/// Pumps the backend every 20 ms, like the desktop's pump thread, until +/// dropped. +struct Pump(Arc); + +impl Pump { + fn start(backend: Arc) -> Self { + let stop = Arc::new(AtomicBool::new(false)); + let flag = stop.clone(); + std::thread::spawn(move || { + while !flag.load(Ordering::Relaxed) { + backend.pump(); + std::thread::sleep(Duration::from_millis(20)); + } + }); + Self(stop) + } +} + +impl Drop for Pump { + fn drop(&mut self) { + self.0.store(true, Ordering::Relaxed); + } +} + +#[test] +fn find_streams_results_then_done_and_cancels() { + let dir = tempfile::tempdir().unwrap(); + let tree = lutra_testkit::generate_search_tree(dir.path(), 1_000); + let backend = Arc::new(EngineBackend::new()); + let server = lutra_control::start(0, backend.clone()).unwrap(); + let mut client = Client::connect(server.local_addr()); + client.call(json!({"id": 1, "method": "events.subscribe"})); + let r = client.call(execute("project.open", json!({"path": dir.path()}))); + assert!(r["result"]["root"].is_string(), "{r}"); + let _pump = Pump::start(backend.clone()); + + let r = client.call(execute( + "search.find", + json!({"query": lutra_testkit::NEEDLE, "caseSensitive": true}), + )); + let id = r["result"]["searchId"].as_u64().expect("searchId"); + let (results, done) = client.until_done(id); + assert_eq!(results.len(), tree.needle_files); + for result in &results { + let m = &result["matches"][0]; + assert!(m["line"].is_u64() && m["column"].is_u64(), "{result}"); + assert!( + m["preview"] + .as_str() + .unwrap() + .contains(lutra_testkit::NEEDLE) + ); + assert!(m["previewColumn"].is_u64()); + let path = result["path"].as_str().unwrap(); + assert!(!path.starts_with("var/") && !path.contains('\\'), "{path}"); + } + assert_eq!(done["files"], tree.needle_files); + assert_eq!(done["cancelled"], false); + assert_eq!(done["truncated"], false); + assert!(done["elapsedMs"].is_u64()); + + let r = client.call(execute("search.find", json!({"query": "(", "regex": true}))); + assert_eq!(r["error"]["code"], "invalid_params", "{r}"); + + let r = client.call(execute( + "search.find", + json!({"query": "function", "maxResults": 100000}), + )); + let id = r["result"]["searchId"].as_u64().unwrap(); + let r = client.call(execute("search.cancel", json!({"searchId": id}))); + assert!(r["result"]["cancelled"].is_boolean(), "{r}"); + let (_, done) = client.until_done(id); + assert_eq!(done["kind"], "searchDone"); + assert!( + done["cancelled"] == true || r["result"]["cancelled"] == false, + "{done}" + ); +} diff --git a/crates/engine/Cargo.toml b/crates/engine/Cargo.toml index 80c5bf1..ec1f13c 100644 --- a/crates/engine/Cargo.toml +++ b/crates/engine/Cargo.toml @@ -12,6 +12,7 @@ publish.workspace = true layer = 4 [dependencies] +lutra-search.workspace = true lutra-text.workspace = true lutra-vfs.workspace = true serde = { version = "1.0.229", features = ["derive"] } @@ -22,5 +23,6 @@ thiserror = "2.0.21" workspace = true [dev-dependencies] +lutra-testkit.workspace = true proptest.workspace = true tempfile.workspace = true diff --git a/crates/engine/src/cmd/mod.rs b/crates/engine/src/cmd/mod.rs index 5ea5498..0c24c0c 100644 --- a/crates/engine/src/cmd/mod.rs +++ b/crates/engine/src/cmd/mod.rs @@ -6,6 +6,7 @@ mod file; mod params; mod project; mod recovery; +pub mod search; mod settings; pub mod theme; pub mod ui; @@ -39,5 +40,7 @@ pub fn register_builtins(registry: &mut Registry) -> Result<(), EngineError> { registry.register(Box::new(theme::List))?; registry.register(Box::new(theme::SystemPreference))?; registry.register(Box::new(ui::RegisterCommands))?; + registry.register(Box::new(search::Find))?; + registry.register(Box::new(search::Cancel))?; Ok(()) } diff --git a/crates/engine/src/cmd/search.rs b/crates/engine/src/cmd/search.rs new file mode 100644 index 0000000..91c92c2 --- /dev/null +++ b/crates/engine/src/cmd/search.rs @@ -0,0 +1,304 @@ +//! Find in files: `search.find` and `search.cancel`. +//! +//! `search.find` compiles the query, takes a snapshot of the dirty buffers +//! and starts the search on its own threads ([`lutra_search::spawn`]), then +//! returns `{searchId}` at once; the engine lock is never held while files +//! are read. Results wait in a channel per search until +//! [`crate::Engine::pump_vfs`] calls [`pump`], which turns them into +//! [`Event::SearchResults`] (one per file) and a final +//! [`Event::SearchDone`]. Hosts already publish the pump's events, so +//! results reach the window and control-channel subscribers in the +//! engine's event order, at most one batch per pump (every 100 ms on the +//! desktop) and at most [`MAX_RESULTS_PER_PUMP`] files per batch, so a +//! subscriber's queue never overflows. Searches whose project is no longer +//! open are cancelled by the next pump and end with `cancelled: true`. + +use std::collections::BTreeMap; +use std::path::{Component, PathBuf}; +use std::sync::mpsc::{Receiver, TryRecvError}; +use std::time::Instant; + +use lutra_search::{CancelToken, Overlay, Query, Search, SearchEvent}; +use serde::Deserialize; +use serde_json::{Value, json}; + +use crate::cmd::params::parse; +use crate::{Command, CommandSpec, EngineError, Event, Outcome, SearchMatch, State}; + +/// Most `searchResults` events one pump emits, over all searches; the rest +/// wait for the next pump. +pub const MAX_RESULTS_PER_PUMP: usize = 500; + +/// Most include or exclude globs one query may carry. +pub const MAX_GLOBS: usize = 256; + +/// The running searches. +#[derive(Debug, Default)] +pub struct Searches { + /// The id of the last search started; ids are never reused. + last_id: u64, + /// Searches whose `searchDone` has not been emitted yet, by id. + running: BTreeMap, +} + +impl Searches { + /// Number of searches whose `searchDone` is still to come. + pub fn running(&self) -> usize { + self.running.len() + } +} + +/// One search in flight. +#[derive(Debug)] +struct Running { + /// The project root the search walks. + root: PathBuf, + /// Stops it. + cancel: CancelToken, + /// Its results and summary. + events: Receiver, + /// When it started, for a summary the search thread never sent. + started: Instant, + /// Files emitted so far. + files: usize, + /// Matches emitted so far. + matches: usize, +} + +/// Params of `search.find`. +#[derive(Deserialize)] +#[serde(deny_unknown_fields, rename_all = "camelCase")] +struct FindParams { + /// The text or regular expression. + query: String, + /// `query` is a regular expression. + #[serde(default)] + regex: bool, + /// Match letter case exactly. + #[serde(default)] + case_sensitive: bool, + /// Whole words only. + #[serde(default)] + whole_word: bool, + /// Only files matching one of these globs. + #[serde(default)] + include: Vec, + /// Skip files and directories matching one of these globs. + #[serde(default)] + exclude: Vec, + /// Cap on matches. + max_results: Option, +} + +/// `search.find`: starts a search in the project. +pub struct Find; + +impl Command for Find { + fn spec(&self) -> &CommandSpec { + &CommandSpec { + id: "search.find", + label: "Find in Files", + menu_path: None, + default_shortcut: None, + params: "{ query, regex?, caseSensitive?, wholeWord?, include?: [glob], exclude?: [glob], maxResults? (default 10000, at most 100000) }; returns { searchId } at once, then events searchResults { searchId, path, matches: [{ line, column, length, preview, previewColumn }] } per file and searchDone { searchId, files, matches, truncated, cancelled, elapsedMs }; invalid_params for an empty query, a bad regex or glob", + } + } + + fn enabled(&self, state: &State) -> bool { + state.has_project() + } + + fn run(&self, state: &mut State, params: Value) -> Result { + let id = self.spec().id; + let p: FindParams = parse(id, params)?; + if p.include.len() > MAX_GLOBS || p.exclude.len() > MAX_GLOBS { + return Err(EngineError::invalid_params( + id, + format!("at most {MAX_GLOBS} include and {MAX_GLOBS} exclude globs"), + )); + } + let query = Query { + pattern: p.query, + regex: p.regex, + case_sensitive: p.case_sensitive, + whole_word: p.whole_word, + include: p.include, + exclude: p.exclude, + max_results: p.max_results.unwrap_or(lutra_search::DEFAULT_MAX_RESULTS), + }; + let search = + Search::new(&query).map_err(|e| EngineError::invalid_params(id, e.to_string()))?; + let root = state + .project + .as_ref() + .ok_or_else(|| EngineError::Disabled(id.to_string()))? + .project + .root() + .to_path_buf(); + let cancel = CancelToken::new(); + let events = lutra_search::spawn(root.clone(), search, overlay(state), cancel.clone()) + .map_err(|e| EngineError::Failed(format!("cannot start the search: {e}")))?; + let searches = &mut state.searches; + searches.last_id = searches.last_id.saturating_add(1); + let search_id = searches.last_id; + searches.running.insert( + search_id, + Running { + root, + cancel, + events, + started: Instant::now(), + files: 0, + matches: 0, + }, + ); + Ok(Outcome::value(json!({ "searchId": search_id }))) + } +} + +/// Params of `search.cancel`. +#[derive(Deserialize)] +#[serde(deny_unknown_fields, rename_all = "camelCase")] +struct CancelParams { + /// The search to stop. + search_id: u64, +} + +/// `search.cancel`: stops a running search. +pub struct Cancel; + +impl Command for Cancel { + fn spec(&self) -> &CommandSpec { + &CommandSpec { + id: "search.cancel", + label: "Cancel Search", + menu_path: None, + default_shortcut: None, + params: "{ searchId }; returns { cancelled }: false if the search already ended or never existed; a cancelled search still ends with searchDone (cancelled: true)", + } + } + + fn run(&self, state: &mut State, params: Value) -> Result { + let p: CancelParams = parse(self.spec().id, params)?; + let running = state.searches.running.get(&p.search_id); + if let Some(running) = running { + running.cancel.cancel(); + } + Ok(Outcome::value(json!({ "cancelled": running.is_some() }))) + } +} + +/// The text of every dirty buffer by project-relative path, searched +/// instead of the file on disk. A path that is not plain Unicode cannot +/// match a walked file and is left out. +fn overlay(state: &State) -> Overlay { + state + .buffers + .dirty_ids() + .into_iter() + .filter_map(|id| state.buffers.get(id)) + .filter_map(|b| Some((slash_path(&b.path)?, b.buffer.text()))) + .collect() +} + +/// A project-relative path with `/` separators. +fn slash_path(path: &std::path::Path) -> Option { + let parts: Option> = path + .components() + .map(|c| match c { + Component::Normal(name) => name.to_str(), + _ => None, + }) + .collect(); + Some(parts?.join("/")) +} + +/// Moves finished results of running searches into events: at most +/// [`MAX_RESULTS_PER_PUMP`] `searchResults`, and `searchDone` for every +/// search that ended. Searches of a project that is no longer open are +/// cancelled; their remaining results are dropped. A search thread that +/// vanished without a summary ends with `cancelled: true` and the counts +/// emitted so far. +pub fn pump(state: &mut State) { + let root = state + .project + .as_ref() + .map(|w| w.project.root().to_path_buf()); + let mut budget = MAX_RESULTS_PER_PUMP; + let mut events = Vec::new(); + let mut finished = Vec::new(); + for (&search_id, running) in state.searches.running.iter_mut() { + let stale = root.as_ref() != Some(&running.root); + if stale { + running.cancel.cancel(); + } + loop { + if budget == 0 && !stale { + break; + } + match running.events.try_recv() { + Ok(SearchEvent::File(file)) => { + if stale { + continue; + } + budget = budget.saturating_sub(1); + running.files = running.files.saturating_add(1); + running.matches = running.matches.saturating_add(file.matches.len()); + events.push(Event::SearchResults { + search_id, + path: file.path, + matches: file.matches.into_iter().map(SearchMatch::from).collect(), + }); + } + Ok(SearchEvent::Done(summary)) => { + events.push(Event::SearchDone { + search_id, + files: summary.files, + matches: summary.matches, + truncated: summary.truncated, + cancelled: summary.cancelled || stale, + elapsed_ms: millis(summary.elapsed), + }); + finished.push(search_id); + break; + } + Err(TryRecvError::Empty) => break, + Err(TryRecvError::Disconnected) => { + events.push(Event::SearchDone { + search_id, + files: running.files, + matches: running.matches, + truncated: false, + cancelled: true, + elapsed_ms: millis(running.started.elapsed()), + }); + finished.push(search_id); + break; + } + } + } + } + for search_id in finished { + state.searches.running.remove(&search_id); + } + for event in events { + state.emit(event); + } +} + +/// Whole milliseconds of `elapsed`, saturating. +fn millis(elapsed: std::time::Duration) -> u64 { + u64::try_from(elapsed.as_millis()).unwrap_or(u64::MAX) +} + +impl From for SearchMatch { + fn from(m: lutra_search::Match) -> Self { + Self { + line: m.line, + column: m.column, + length: m.length, + preview: m.preview, + preview_column: m.preview_column, + } + } +} diff --git a/crates/engine/src/engine.rs b/crates/engine/src/engine.rs index e9f42d7..66ecba7 100644 --- a/crates/engine/src/engine.rs +++ b/crates/engine/src/engine.rs @@ -130,7 +130,8 @@ impl Engine { /// Applies pending changes on disk (reloads clean buffers, reports /// changed directories and conflicts), writes due recovery-journal - /// entries (see [`crate::journal`]) and returns every queued event. + /// entries (see [`crate::journal`]), moves finished search results into + /// events (see [`cmd::search`]) and returns every queued event. /// Hosts call it regularly, e.g. every 100 ms from a background thread. /// /// # Errors @@ -141,6 +142,7 @@ impl Engine { guard::run(|| { crate::workspace::pump(state); crate::journal::tick(state, std::time::Instant::now()); + cmd::search::pump(state); Ok(()) })?; Ok(state.take_events()) @@ -373,6 +375,8 @@ mod tests { "recovery.discard", "recovery.list", "recovery.restore", + "search.cancel", + "search.find", "settings.get", "settings.list", "settings.reset", diff --git a/crates/engine/src/event.rs b/crates/engine/src/event.rs index 0d7ae8c..fc8e6c6 100644 --- a/crates/engine/src/event.rs +++ b/crates/engine/src/event.rs @@ -144,6 +144,49 @@ pub enum Event { /// Project-relative paths with `/`, sorted. paths: Vec, }, + /// Matches of a running search (`search.find`) in one file. A file is + /// reported once per search; files arrive in no particular order. + SearchResults { + /// The search, as `search.find` returned it. + search_id: u64, + /// Project-relative path of the file. + path: String, + /// The matches in line order; never empty. + matches: Vec, + }, + /// A search ended; no `searchResults` for it follow. + SearchDone { + /// The search. + search_id: u64, + /// Files with reported matches. + files: usize, + /// Matches reported. + matches: usize, + /// The match cap stopped the search; more matches exist. + truncated: bool, + /// `search.cancel` (or closing the project) stopped the search. + cancelled: bool, + /// Wall time of the search in milliseconds. + elapsed_ms: u64, + }, +} + +/// One match of [`Event::SearchResults`]. Positions are 0-based; columns +/// and lengths count UTF-16 code units. +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct SearchMatch { + /// Line of the match. + pub line: usize, + /// Start of the match in the line. + pub column: usize, + /// Length of the match; never 0. + pub length: usize, + /// The line without its line break, or a window of at most 300 + /// characters around the match for longer lines. + pub preview: String, + /// Start of the match inside `preview`. + pub preview_column: usize, } /// Severity of an [`Event::Notification`]. @@ -224,6 +267,39 @@ mod tests { ); } + #[test] + fn search_shapes() { + let e = Event::SearchResults { + search_id: 2, + path: "src/a.php".into(), + matches: vec![super::SearchMatch { + line: 0, + column: 6, + length: 4, + preview: ", /// UI commands the frontend registered (`ui.registerCommands`), by id. pub ui_commands: BTreeMap, + /// Running searches (`search.find`). + pub searches: crate::cmd::search::Searches, /// Number of notifications emitted, for their ids. notifications: u64, /// Events queued for the host, oldest first. diff --git a/crates/engine/tests/search.rs b/crates/engine/tests/search.rs new file mode 100644 index 0000000..9babc0b --- /dev/null +++ b/crates/engine/tests/search.rs @@ -0,0 +1,207 @@ +//! `search.find` and `search.cancel`: results arrive through the pump. + +#![allow( + clippy::unwrap_used, + clippy::expect_used, + clippy::panic, + clippy::indexing_slicing +)] + +use std::fs; +use std::time::{Duration, Instant}; + +use lutra_engine::cmd::search::MAX_RESULTS_PER_PUMP; +use lutra_engine::{Engine, Event}; +use serde_json::{Value, json}; +use tempfile::TempDir; + +/// A temp folder with `files`, opened in a fresh engine. +fn open_project(files: &[(&str, &str)]) -> (TempDir, Engine) { + let dir = tempfile::tempdir().unwrap(); + for (path, text) in files { + let abs = dir.path().join(path); + fs::create_dir_all(abs.parent().unwrap()).unwrap(); + fs::write(abs, text).unwrap(); + } + let mut engine = Engine::new().unwrap(); + engine + .execute("project.open", json!({"path": dir.path()})) + .unwrap(); + (dir, engine) +} + +fn find(engine: &mut Engine, params: Value) -> u64 { + let out = engine.execute("search.find", params).unwrap(); + assert!(out.events.is_empty(), "results only come from the pump"); + out.value["searchId"].as_u64().unwrap() +} + +/// Pumps until `searchDone` for `id`; returns the results (sorted by path) +/// and the done event, and the number of pumps. +fn collect(engine: &mut Engine, id: u64) -> (Vec, Event, usize) { + let deadline = Instant::now() + Duration::from_secs(10); + let mut results = Vec::new(); + let mut pumps = 0; + while Instant::now() < deadline { + pumps += 1; + for event in engine.pump_vfs().unwrap() { + match &event { + Event::SearchResults { search_id, .. } if *search_id == id => results.push(event), + Event::SearchDone { search_id, .. } if *search_id == id => { + results.sort_by_key(path); + return (results, event, pumps); + } + _ => {} + } + } + std::thread::sleep(Duration::from_millis(10)); + } + panic!("no searchDone within 10 s"); +} + +fn path(event: &Event) -> String { + match event { + Event::SearchResults { path, .. } => path.clone(), + _ => String::new(), + } +} + +#[test] +fn finds_in_files_and_dirty_buffers() { + let (_dir, mut engine) = open_project(&[ + ("src/a.php", " = results.iter().map(path).collect(); + assert_eq!(paths, ["src/a.php", "src/b.php", "vendor/x/c.php"]); + let Event::SearchResults { matches, .. } = &results[1] else { + panic!() + }; + assert_eq!( + (matches[0].line, matches[0].column, matches[0].length), + (1, 1, 6) + ); + assert_eq!(matches[0].preview, "$needle = 1;"); + let Event::SearchDone { + files, + matches, + truncated, + cancelled, + .. + } = done + else { + panic!() + }; + assert_eq!((files, matches, truncated, cancelled), (3, 3, false, false)); + + let id2 = find( + &mut engine, + json!({"query": "needle", "caseSensitive": true, "include": ["src/*.php"]}), + ); + assert!(id2 > id); + let (results, _, _) = collect(&mut engine, id2); + assert_eq!(results.len(), 2); +} + +#[test] +fn invalid_queries_fail_with_invalid_params() { + let (_dir, mut engine) = open_project(&[("a.php", "x")]); + for params in [ + json!({"query": "(", "regex": true}), + json!({"query": ""}), + json!({"query": "x", "include": ["["]}), + json!({"query": "x", "bogus": 1}), + json!({}), + ] { + let err = engine.execute("search.find", params.clone()).unwrap_err(); + assert_eq!(err.code(), "invalid_params", "{params}"); + } + assert_eq!(engine.state().searches.running(), 0); + engine.execute("project.close", json!({})).unwrap(); + let err = engine + .execute("search.find", json!({"query": "x"})) + .unwrap_err(); + assert_eq!(err.code(), "disabled"); +} + +#[test] +fn cancel_and_project_close_end_searches() { + let dir = tempfile::tempdir().unwrap(); + lutra_testkit::generate_search_tree(dir.path(), 2_000); + let mut engine = Engine::new().unwrap(); + engine + .execute("project.open", json!({"path": dir.path()})) + .unwrap(); + + let id = find(&mut engine, json!({"query": "function"})); + let out = engine + .execute("search.cancel", json!({"searchId": id})) + .unwrap(); + assert_eq!(out.value["cancelled"], true); + let (_, done, _) = collect(&mut engine, id); + let Event::SearchDone { + cancelled, matches, .. + } = done + else { + panic!() + }; + assert!( + cancelled || matches == 10_000, + "cancelled unless it had already finished" + ); + let out = engine + .execute("search.cancel", json!({"searchId": id})) + .unwrap(); + assert_eq!(out.value["cancelled"], false); + + let id = find(&mut engine, json!({"query": "function"})); + engine.execute("project.close", json!({})).unwrap(); + let (results, done, _) = collect(&mut engine, id); + assert!(results.is_empty()); + assert!(matches!( + done, + Event::SearchDone { + cancelled: true, + .. + } + )); + assert_eq!(engine.state().searches.running(), 0); +} + +#[test] +fn pumps_spread_many_results() { + let dir = tempfile::tempdir().unwrap(); + for i in 0..(MAX_RESULTS_PER_PUMP + 100) { + fs::write(dir.path().join(format!("f{i}.txt")), "hit\n").unwrap(); + } + let mut engine = Engine::new().unwrap(); + engine + .execute("project.open", json!({"path": dir.path()})) + .unwrap(); + let id = find(&mut engine, json!({"query": "hit"})); + std::thread::sleep(Duration::from_millis(500)); + let first = engine.pump_vfs().unwrap(); + let batch = first + .iter() + .filter(|e| matches!(e, Event::SearchResults { .. })) + .count(); + assert!(batch <= MAX_RESULTS_PER_PUMP, "{batch}"); + let (rest, done, _) = collect(&mut engine, id); + assert_eq!(batch + rest.len(), MAX_RESULTS_PER_PUMP + 100); + assert!(matches!(done, Event::SearchDone { files, .. } if files == MAX_RESULTS_PER_PUMP + 100)); +} diff --git a/crates/search/Cargo.toml b/crates/search/Cargo.toml index 736b8b9..4c4417f 100644 --- a/crates/search/Cargo.toml +++ b/crates/search/Cargo.toml @@ -12,6 +12,17 @@ publish.workspace = true layer = 2 [dependencies] +globset = "0.4.18" +grep-matcher = "0.1.7" +grep-regex = "0.1.13" +grep-searcher = "0.1.14" +ignore.workspace = true +thiserror = "2.0.21" + +[dev-dependencies] +lutra-testkit.workspace = true +proptest.workspace = true +tempfile.workspace = true [lints] workspace = true diff --git a/crates/search/src/lib.rs b/crates/search/src/lib.rs index e623569..4f5b1a2 100644 --- a/crates/search/src/lib.rs +++ b/crates/search/src/lib.rs @@ -2,3 +2,234 @@ //! //! Streams matches from the project files off the UI thread, honouring the //! ignore rules of the project model. +//! +//! # Query +//! A [`Query`] is compiled once into a [`Search`] ([`Search::new`]): a +//! literal or a regular expression (Rust `regex` syntax), case sensitive or +//! not, optionally whole words only, plus include and exclude globs. An +//! invalid pattern or glob fails with [`SearchError`] before anything runs. +//! Patterns never match across lines. +//! +//! # Which files are searched +//! The walk follows the rules of `lutra-vfs`: +//! - `.gitignore`, `.ignore`, `.git/info/exclude` and the global git excludes +//! apply whether or not the folder is a git repository; ignored files and +//! everything below an ignored directory are skipped. Ignore files above +//! the project root are not read. +//! - `.git` (directory or file) at any depth and every `node_modules` +//! directory are skipped. +//! - `vendor/` directly under the root is library code and is searched even +//! when an ignore file lists it (Composer projects usually do); ignore +//! files inside it still apply. +//! - Hidden (dot) files are searched, as the project tree shows them. +//! Symbolic links are not followed. +//! - Include globs keep only files that match one of them, exclude globs +//! drop files and whole directories. A glob without `/` matches the file +//! or directory name (`*.php`), a glob with `/` the project-relative path +//! (`src/**/*.php`). +//! - Files larger than [`MAX_FILE_SIZE`], binary files (a NUL byte) and +//! files that cannot be read are skipped. +//! +//! # Results +//! Each file with matches is reported once as [`FileMatches`], in no +//! particular order (the walk is parallel). Positions are 0-based; columns +//! and lengths count UTF-16 code units. Lines end at `\n` (a `\r` before it +//! is not part of the line). The preview is the line, or a window of at most +//! [`MAX_PREVIEW_CHARS`] characters around the match for longer lines. +//! At most [`Query::max_results`] matches are reported in total; the +//! [`Summary`] says whether the cap cut the search short. +//! +//! # Threads and cancellation +//! [`run`] blocks the calling thread and searches on a pool of up to +//! [`MAX_THREADS`] walker threads; [`spawn`] runs it on a thread of its own +//! and delivers [`SearchEvent`]s over a channel. A [`CancelToken`] stops a +//! running search between files and between lines. +//! +//! # Open buffers +//! An [`Overlay`] maps project-relative paths to text that replaces the file +//! on disk, so unsaved edits are searched instead of the stale file. Only +//! files the walk visits are looked up; text for a path that is not on disk +//! (or is skipped by the rules) is not searched. + +mod lines; +mod query; +mod walk; + +use std::collections::HashMap; +use std::path::PathBuf; +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::mpsc::{Receiver, channel}; +use std::time::Duration; + +pub use query::Search; +pub use walk::run; + +/// Largest file searched, in bytes (8 MiB). +pub const MAX_FILE_SIZE: u64 = 8 * 1024 * 1024; + +/// Default cap on the number of matches of one search. +pub const DEFAULT_MAX_RESULTS: usize = 10_000; + +/// Highest cap a query may ask for. +pub const MAX_RESULTS_LIMIT: usize = 100_000; + +/// Longest preview, in characters (Unicode scalar values). +pub const MAX_PREVIEW_CHARS: usize = 300; + +/// Most walker threads of one search. +pub const MAX_THREADS: usize = 8; + +/// What to search for and where. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Query { + /// The text or regular expression; must not be empty. + pub pattern: String, + /// Treat `pattern` as a regular expression instead of literal text. + pub regex: bool, + /// Match letter case exactly. + pub case_sensitive: bool, + /// Only matches that start and end at word boundaries. + pub whole_word: bool, + /// Only files matching one of these globs; all files when empty. + pub include: Vec, + /// Skip files and directories matching one of these globs. + pub exclude: Vec, + /// Stop after this many matches; clamped to `1..=`[`MAX_RESULTS_LIMIT`]. + pub max_results: usize, +} + +impl Query { + /// A literal, case-insensitive query for `pattern` with the default cap. + pub fn literal(pattern: impl Into) -> Self { + Self { + pattern: pattern.into(), + regex: false, + case_sensitive: false, + whole_word: false, + include: Vec::new(), + exclude: Vec::new(), + max_results: DEFAULT_MAX_RESULTS, + } + } +} + +/// Why a query cannot run. +#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +pub enum SearchError { + /// The pattern is empty. + #[error("the search text is empty")] + EmptyPattern, + /// The pattern is not a valid regular expression, or matches a line + /// break. + #[error("invalid pattern: {0}")] + InvalidPattern(String), + /// An include or exclude glob is invalid. + #[error("invalid glob: {0}")] + InvalidGlob(String), +} + +/// One match. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Match { + /// 0-based line. + pub line: usize, + /// Start in the line, in UTF-16 code units. + pub column: usize, + /// Length in UTF-16 code units; never 0. + pub length: usize, + /// The line, or a window of it around the match, without the line break. + pub preview: String, + /// Start of the match inside `preview`, in UTF-16 code units. + pub preview_column: usize, +} + +/// The matches in one file, in line order. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct FileMatches { + /// Project-relative path with `/` separators. + pub path: String, + /// The matches; never empty. + pub matches: Vec, +} + +/// How a search ended. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub struct Summary { + /// Files with at least one reported match. + pub files: usize, + /// Matches reported. + pub matches: usize, + /// The match cap stopped the search; more matches may exist. + pub truncated: bool, + /// The search was cancelled before it finished. + pub cancelled: bool, + /// Wall time from start to end. + pub elapsed: Duration, +} + +/// What a search started with [`spawn`] delivers, in order: any number of +/// files, then exactly one summary. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum SearchEvent { + /// A file with matches. + File(FileMatches), + /// The search ended; nothing follows. + Done(Summary), +} + +/// Text of open buffers by project-relative path (`/` separators), searched +/// instead of the file on disk. +pub type Overlay = HashMap; + +/// Stops a running search. Clones share the flag. +#[derive(Debug, Clone, Default)] +pub struct CancelToken(Arc); + +impl CancelToken { + /// A token that is not cancelled. + pub fn new() -> Self { + Self::default() + } + + /// Asks the search to stop; it ends with [`Summary::cancelled`] set. + pub fn cancel(&self) { + self.0.store(true, Ordering::Relaxed); + } + + /// Whether [`CancelToken::cancel`] was called. + pub fn is_cancelled(&self) -> bool { + self.0.load(Ordering::Relaxed) + } +} + +/// Runs [`run`] on a new thread and returns the channel of its events. A +/// receiver that is dropped does not stop the search; cancel it with +/// `cancel`. +/// +/// # Parameters +/// - `root`: the project root (absolute). +/// - `search`: the compiled query. +/// - `overlay`: buffer texts that replace files on disk. +/// - `cancel`: stops the search. +/// +/// # Errors +/// The OS refused to start the thread. +pub fn spawn( + root: PathBuf, + search: Search, + overlay: Overlay, + cancel: CancelToken, +) -> std::io::Result> { + let (tx, rx) = channel(); + std::thread::Builder::new() + .name("lutra-search".into()) + .spawn(move || { + let files = tx.clone(); + let summary = run(&root, &search, &overlay, &cancel, &move |file| { + let _receiver_gone = files.send(SearchEvent::File(file)); + }); + let _receiver_gone = tx.send(SearchEvent::Done(summary)); + })?; + Ok(rx) +} diff --git a/crates/search/src/lines.rs b/crates/search/src/lines.rs new file mode 100644 index 0000000..b0a4469 --- /dev/null +++ b/crates/search/src/lines.rs @@ -0,0 +1,195 @@ +//! Turning a matching line into [`Match`]es: UTF-16 columns and previews. + +use grep_matcher::Matcher; +use grep_regex::RegexMatcher; + +use crate::{MAX_PREVIEW_CHARS, Match}; + +/// Characters shown before the match when a long line is cut to a window. +const PREVIEW_LEAD: usize = 80; + +/// Appends the matches of one line to `out`, at most `limit` in total. +/// +/// # Parameters +/// - `matcher`: the compiled pattern. +/// - `line`: 0-based line number. +/// - `bytes`: the line as read, with or without its line break. Invalid +/// UTF-8 is replaced by U+FFFD before matching, so positions refer to the +/// text an editor shows. +/// - `out`: receives the matches; zero-length matches are skipped. +/// - `limit`: stop once `out` holds this many matches. +/// +/// A matcher error (none is expected for an in-memory regex) ends the line +/// early; the matches found so far are kept. +pub(crate) fn collect( + matcher: &RegexMatcher, + line: usize, + bytes: &[u8], + out: &mut Vec, + limit: usize, +) { + let bytes = trim_line_break(bytes); + let text = String::from_utf8_lossy(bytes); + let mut spans = Vec::new(); + let _matcher_error = matcher.find_iter(text.as_bytes(), |m| { + if m.start() < m.end() { + spans.push((m.start(), m.end())); + } + out.len().saturating_add(spans.len()) < limit + }); + if spans.is_empty() { + return; + } + let window = Window::new(&text); + let mut cursor = Cursor::default(); + for (start, end) in spans { + let Some(matched) = text.get(start..end) else { + continue; + }; + if !cursor.advance(&text, start) { + continue; + } + let (preview, preview_column) = window.preview(&text, &cursor, start); + out.push(Match { + line, + column: cursor.utf16, + length: utf16_len(matched), + preview, + preview_column, + }); + } +} + +/// `bytes` without a trailing `\n` or `\r\n`. +fn trim_line_break(bytes: &[u8]) -> &[u8] { + let bytes = bytes.strip_suffix(b"\n").unwrap_or(bytes); + bytes.strip_suffix(b"\r").unwrap_or(bytes) +} + +/// Number of UTF-16 code units of `s`. +fn utf16_len(s: &str) -> usize { + s.chars().map(char::len_utf16).sum() +} + +/// A position in the line, counted three ways; only moves forward. +#[derive(Default)] +struct Cursor { + /// Byte offset. + byte: usize, + /// Characters before `byte`. + chars: usize, + /// UTF-16 code units before `byte`. + utf16: usize, +} + +impl Cursor { + /// Moves to byte offset `to`; false if `to` lies behind the cursor or + /// not on a character boundary. + fn advance(&mut self, text: &str, to: usize) -> bool { + let Some(between) = text.get(self.byte..to) else { + return false; + }; + for c in between.chars() { + self.chars = self.chars.saturating_add(1); + self.utf16 = self.utf16.saturating_add(c.len_utf16()); + } + self.byte = to; + true + } +} + +/// How previews of one line are cut. +enum Window { + /// The line is short enough to be the preview of every match. + Whole, + /// The line is longer than [`MAX_PREVIEW_CHARS`]: the byte offset of + /// every character, plus the text length at the end. + Cut(Vec), +} + +impl Window { + /// Prepares previews for `text`. + fn new(text: &str) -> Self { + if text.len() <= MAX_PREVIEW_CHARS || text.chars().count() <= MAX_PREVIEW_CHARS { + return Self::Whole; + } + let mut offsets: Vec = text.char_indices().map(|(i, _)| i).collect(); + offsets.push(text.len()); + Self::Cut(offsets) + } + + /// The preview for a match at `cursor` (byte offset `start`), and the + /// match's UTF-16 column inside it. + fn preview(&self, text: &str, cursor: &Cursor, start: usize) -> (String, usize) { + let Self::Cut(offsets) = self else { + return (text.to_string(), cursor.utf16); + }; + let total = offsets.len().saturating_sub(1); + let first = cursor + .chars + .saturating_sub(PREVIEW_LEAD) + .min(total.saturating_sub(MAX_PREVIEW_CHARS)); + let last = first.saturating_add(MAX_PREVIEW_CHARS).min(total); + let from = offsets.get(first).copied().unwrap_or(0); + let to = offsets.get(last).copied().unwrap_or(text.len()); + let preview = text.get(from..to).unwrap_or_default().to_string(); + let column = text.get(from..start).map(utf16_len).unwrap_or(0); + (preview, column) + } +} + +#[cfg(test)] +mod tests { + use grep_regex::RegexMatcherBuilder; + + use super::*; + + fn matcher(pattern: &str) -> RegexMatcher { + RegexMatcherBuilder::new() + .fixed_strings(true) + .line_terminator(Some(b'\n')) + .build(pattern) + .unwrap() + } + + #[test] + fn columns_count_utf16_units() { + let mut out = Vec::new(); + collect(&matcher("x"), 4, "รค๐Ÿ˜€x and x\r\n".as_bytes(), &mut out, 10); + assert_eq!(out.len(), 2); + assert_eq!((out[0].line, out[0].column, out[0].length), (4, 3, 1)); + assert_eq!(out[0].preview, "รค๐Ÿ˜€x and x"); + assert_eq!(out[0].preview_column, 3); + assert_eq!(out[1].column, 9); + } + + #[test] + fn long_lines_are_cut_around_the_match() { + let line = format!("{}needle{}", "a".repeat(1000), "b".repeat(1000)); + let mut out = Vec::new(); + collect(&matcher("needle"), 0, line.as_bytes(), &mut out, 10); + let m = &out[0]; + assert_eq!(m.column, 1000); + assert_eq!(m.preview.chars().count(), MAX_PREVIEW_CHARS); + assert_eq!( + m.preview.get(m.preview_column..m.preview_column + 6), + Some("needle") + ); + assert_eq!(m.preview_column, PREVIEW_LEAD); + + let tail = format!("{}needle", "a".repeat(1000)); + out.clear(); + collect(&matcher("needle"), 0, tail.as_bytes(), &mut out, 10); + assert!(out[0].preview.ends_with("needle")); + assert_eq!(out[0].preview.chars().count(), MAX_PREVIEW_CHARS); + } + + #[test] + fn invalid_utf8_and_limits() { + let mut out = Vec::new(); + collect(&matcher("ab"), 0, b"\xffab ab ab\n", &mut out, 2); + assert_eq!(out.len(), 2); + assert_eq!(out[0].column, 1); + assert_eq!(out[0].preview, "\u{fffd}ab ab ab"); + } +} diff --git a/crates/search/src/query.rs b/crates/search/src/query.rs new file mode 100644 index 0000000..5cfe997 --- /dev/null +++ b/crates/search/src/query.rs @@ -0,0 +1,139 @@ +//! Compiling a [`Query`]: the matcher and the path filters. + +use globset::{GlobBuilder, GlobSet, GlobSetBuilder}; +use grep_regex::{RegexMatcher, RegexMatcherBuilder}; + +use crate::{MAX_RESULTS_LIMIT, Query, SearchError}; + +/// Largest compiled regex, in bytes; bigger patterns are rejected instead of +/// using unbounded memory. +const REGEX_SIZE_LIMIT: usize = 16 * 1024 * 1024; + +/// Most nesting a pattern may use. +const REGEX_NEST_LIMIT: u32 = 100; + +/// A compiled query, ready to run. +#[derive(Debug, Clone)] +pub struct Search { + /// Matches one line at a time. + pub(crate) matcher: RegexMatcher, + /// Include and exclude globs. + pub(crate) filter: PathFilter, + /// Cap on reported matches. + pub(crate) max_results: usize, +} + +impl Search { + /// Compiles `query`. + /// + /// # Errors + /// [`SearchError::EmptyPattern`], [`SearchError::InvalidPattern`] for a + /// bad regular expression (or one that can match a line break), + /// [`SearchError::InvalidGlob`] for a bad include or exclude glob. + pub fn new(query: &Query) -> Result { + if query.pattern.is_empty() { + return Err(SearchError::EmptyPattern); + } + let matcher = RegexMatcherBuilder::new() + .fixed_strings(!query.regex) + .case_insensitive(!query.case_sensitive) + .word(query.whole_word) + .line_terminator(Some(b'\n')) + .crlf(true) + .size_limit(REGEX_SIZE_LIMIT) + .dfa_size_limit(REGEX_SIZE_LIMIT) + .nest_limit(REGEX_NEST_LIMIT) + .build(&query.pattern) + .map_err(|e| SearchError::InvalidPattern(e.to_string()))?; + Ok(Self { + matcher, + filter: PathFilter::new(&query.include, &query.exclude)?, + max_results: query.max_results.clamp(1, MAX_RESULTS_LIMIT), + }) + } +} + +/// Include and exclude globs, split by whether they match a name or a path. +#[derive(Debug, Clone)] +pub(crate) struct PathFilter { + /// Include globs; `None` when every file is included. + include: Option, + /// Exclude globs; `None` when nothing is excluded. + exclude: Option, +} + +/// Globs matched against the file name and against the relative path. +#[derive(Debug, Clone)] +struct Globs { + /// Globs without `/`. + names: GlobSet, + /// Globs with `/`. + paths: GlobSet, +} + +impl PathFilter { + /// Compiles the globs; blank ones are ignored. + fn new(include: &[String], exclude: &[String]) -> Result { + Ok(Self { + include: Globs::new(include)?, + exclude: Globs::new(exclude)?, + }) + } + + /// Whether the file at `rel` (project-relative, `/` separators) passes + /// the include and exclude globs. + pub(crate) fn file(&self, rel: &str) -> bool { + let included = self.include.as_ref().is_none_or(|g| g.matches(rel)); + included && !self.excluded(rel) + } + + /// Whether an exclude glob matches the file or directory at `rel`. + pub(crate) fn excluded(&self, rel: &str) -> bool { + self.exclude.as_ref().is_some_and(|g| g.matches(rel)) + } +} + +impl Globs { + /// Compiles `globs`; `None` if there is none. Leading and trailing `/` + /// are dropped, so `tests/` excludes every directory named `tests`. + fn new(globs: &[String]) -> Result, SearchError> { + let mut names = GlobSetBuilder::new(); + let mut paths = GlobSetBuilder::new(); + let mut any = false; + for glob in globs.iter().map(|g| g.trim()).filter(|g| !g.is_empty()) { + let pattern = glob.trim_matches('/'); + if pattern.is_empty() { + continue; + } + let compiled = GlobBuilder::new(pattern) + .literal_separator(true) + .backslash_escape(true) + .build() + .map_err(|e| SearchError::InvalidGlob(e.to_string()))?; + if pattern.contains('/') { + paths.add(compiled); + } else { + names.add(compiled); + } + any = true; + } + if !any { + return Ok(None); + } + let build = |b: GlobSetBuilder| { + b.build() + .map_err(|e| SearchError::InvalidGlob(e.to_string())) + }; + Ok(Some(Self { + names: build(names)?, + paths: build(paths)?, + })) + } + + /// Whether `rel`'s last component matches a name glob or `rel` a path + /// glob. + fn matches(&self, rel: &str) -> bool { + let name = rel.rsplit('/').next().unwrap_or(rel); + self.names.is_match(name) || self.paths.is_match(rel) + } +} diff --git a/crates/search/src/walk.rs b/crates/search/src/walk.rs new file mode 100644 index 0000000..3aef8f4 --- /dev/null +++ b/crates/search/src/walk.rs @@ -0,0 +1,280 @@ +//! The parallel walk over the project and the per-file search. + +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; +use std::time::Instant; + +use grep_matcher::LineTerminator; +use grep_regex::RegexMatcher; +use grep_searcher::{BinaryDetection, Searcher, SearcherBuilder, Sink, SinkMatch}; +use ignore::{DirEntry, WalkBuilder, WalkState}; + +use crate::query::PathFilter; +use crate::{ + CancelToken, FileMatches, MAX_FILE_SIZE, MAX_THREADS, Match, Overlay, Search, Summary, lines, +}; + +/// Searches the project at `root` and calls `on_file` for every file with +/// matches, from the walker threads, while the search runs. Blocks until +/// the walk ends, the cap is reached or `cancel` fires. +/// +/// # Parameters +/// - `root`: the project root (absolute). +/// - `search`: the compiled query. +/// - `overlay`: buffer texts that replace files on disk. +/// - `cancel`: stops the search; checked between files and lines. +/// - `on_file`: receives each file's matches. +/// +/// # Returns +/// The totals. Files that cannot be read are skipped silently: a search +/// reports what it can read, like the project tree lists what it can. +pub fn run( + root: &Path, + search: &Search, + overlay: &Overlay, + cancel: &CancelToken, + on_file: &(dyn Fn(FileMatches) + Sync), +) -> Summary { + let started = Instant::now(); + let totals = Totals::default(); + walker(root, &search.filter).build_parallel().run(|| { + let mut searcher = SearcherBuilder::new() + .line_terminator(LineTerminator::crlf()) + .line_number(true) + .binary_detection(BinaryDetection::quit(0)) + .build(); + let totals = &totals; + Box::new(move |entry| { + if cancel.is_cancelled() || totals.full.load(Ordering::Relaxed) { + return WalkState::Quit; + } + let Ok(entry) = entry else { + return WalkState::Continue; + }; + let Some(rel) = file_path(root, &entry, &search.filter) else { + return WalkState::Continue; + }; + let room = search + .max_results + .saturating_sub(totals.matches.load(Ordering::Relaxed)); + let Some(mut matches) = search_file( + &mut searcher, + &search.matcher, + entry.path(), + overlay.get(&rel), + room, + cancel, + ) else { + return WalkState::Continue; + }; + let granted = totals.reserve(matches.len(), search.max_results); + if granted < matches.len() { + matches.truncate(granted); + totals.full.store(true, Ordering::Relaxed); + } + if granted > 0 { + totals.files.fetch_add(1, Ordering::Relaxed); + on_file(FileMatches { path: rel, matches }); + } + if totals.full.load(Ordering::Relaxed) { + WalkState::Quit + } else { + WalkState::Continue + } + }) + }); + let cancelled = cancel.is_cancelled(); + Summary { + files: totals.files.load(Ordering::Relaxed), + matches: totals.matches.load(Ordering::Relaxed), + truncated: totals.full.load(Ordering::Relaxed) && !cancelled, + cancelled, + elapsed: started.elapsed(), + } +} + +/// Counters shared by the walker threads. +#[derive(Default)] +struct Totals { + /// Files reported. + files: AtomicUsize, + /// Matches reported. + matches: AtomicUsize, + /// The cap was reached, or a file had more matches than fit. + full: AtomicBool, +} + +impl Totals { + /// Takes up to `wanted` matches from what is left under `cap` and + /// returns how many were granted. The caller marks the totals full when + /// it got fewer than it wanted: only then is a match known to be left + /// out. + fn reserve(&self, wanted: usize, cap: usize) -> usize { + let mut used = self.matches.load(Ordering::Relaxed); + loop { + let granted = wanted.min(cap.saturating_sub(used)); + match self.matches.compare_exchange_weak( + used, + used.saturating_add(granted), + Ordering::Relaxed, + Ordering::Relaxed, + ) { + Ok(_) => return granted, + Err(now) => used = now, + } + } + } +} + +/// The walker over `root` with the project's rules (see the crate docs). +fn walker(root: &Path, filter: &PathFilter) -> WalkBuilder { + let threads = std::thread::available_parallelism() + .map(|n| n.get()) + .unwrap_or(2) + .clamp(1, MAX_THREADS); + let mut builder = WalkBuilder::new(root); + builder + .hidden(false) + .parents(false) + .ignore(true) + .git_ignore(true) + .git_exclude(true) + .git_global(true) + .require_git(false) + .follow_links(false) + .threads(threads); + let vendor = root.join("vendor"); + let vendor_is_dir = std::fs::symlink_metadata(&vendor).is_ok_and(|m| m.is_dir()); + if vendor_is_dir { + builder.add(&vendor); + } + let root = root.to_path_buf(); + let filter = filter.clone(); + builder.filter_entry(move |entry| keep_entry(&root, &vendor, &filter, entry)); + builder +} + +/// Whether the walk enters (directory) or visits (file) `entry`: `.git` and +/// `node_modules` are skipped, the root's `vendor` is reached through its +/// own walk root instead of the project root's walk, and exclude globs drop +/// whole directories. +fn keep_entry(root: &Path, vendor: &Path, filter: &PathFilter, entry: &DirEntry) -> bool { + let name = entry.file_name(); + if name == ".git" { + return false; + } + let is_dir = entry.file_type().is_some_and(|t| t.is_dir()); + if is_dir && name == "node_modules" { + return false; + } + if entry.depth() == 1 && entry.path() == vendor { + return false; + } + if is_dir && let Some(rel) = relative(root, entry.path()) { + return !filter.excluded(&rel); + } + true +} + +/// The project-relative path of `entry` if it is a regular file that the +/// globs keep. +fn file_path(root: &Path, entry: &DirEntry, filter: &PathFilter) -> Option { + if !entry.file_type().is_some_and(|t| t.is_file()) { + return None; + } + let rel = relative(root, entry.path())?; + filter.file(&rel).then_some(rel) +} + +/// `path` relative to `root` with `/` separators; `None` outside the root or +/// for a name that is not valid Unicode. +fn relative(root: &Path, path: &Path) -> Option { + let rel: PathBuf = path.strip_prefix(root).ok()?.to_path_buf(); + let parts: Option> = rel.components().map(|c| c.as_os_str().to_str()).collect(); + Some(parts?.join("/")) +} + +/// Searches one file, or the buffer text that replaces it, and returns its +/// matches; `None` without matches, for a binary, oversized or unreadable +/// file, or when cancelled. +/// +/// # Parameters +/// - `room`: most matches to collect; one more is collected when possible, +/// so the caller can tell that the cap cut the file short (with no room +/// left, one match is enough to know that). +fn search_file( + searcher: &mut Searcher, + matcher: &RegexMatcher, + path: &Path, + buffer: Option<&String>, + room: usize, + cancel: &CancelToken, +) -> Option> { + let mut sink = Collect { + matcher, + matches: Vec::new(), + limit: room.saturating_add(1), + binary: false, + cancel, + }; + let result = match buffer { + Some(text) => { + if text.len() as u64 > MAX_FILE_SIZE { + return None; + } + searcher.search_slice(matcher, text.as_bytes(), &mut sink) + } + None => { + let size = std::fs::metadata(path).ok()?.len(); + if size > MAX_FILE_SIZE { + return None; + } + searcher.search_path(matcher, path, &mut sink) + } + }; + if result.is_err() || sink.binary || cancel.is_cancelled() || sink.matches.is_empty() { + return None; + } + Some(sink.matches) +} + +/// Collects the matches of one file. +struct Collect<'a> { + /// The pattern, run again on each matching line to find every match. + matcher: &'a RegexMatcher, + /// Matches so far. + matches: Vec, + /// Stop at this many matches. + limit: usize, + /// The file turned out to be binary. + binary: bool, + /// Stops the file early. + cancel: &'a CancelToken, +} + +impl Sink for Collect<'_> { + type Error = std::io::Error; + + fn matched(&mut self, _searcher: &Searcher, mat: &SinkMatch<'_>) -> Result { + if self.cancel.is_cancelled() { + return Ok(false); + } + let line = mat + .line_number() + .and_then(|n| usize::try_from(n.saturating_sub(1)).ok()) + .unwrap_or(0); + lines::collect( + self.matcher, + line, + mat.bytes(), + &mut self.matches, + self.limit, + ); + Ok(self.matches.len() < self.limit) + } + + fn binary_data(&mut self, _searcher: &Searcher, _offset: u64) -> Result { + self.binary = true; + Ok(false) + } +} diff --git a/crates/search/tests/search.rs b/crates/search/tests/search.rs new file mode 100644 index 0000000..dcbe9cc --- /dev/null +++ b/crates/search/tests/search.rs @@ -0,0 +1,345 @@ +//! Find in files over real directories: matching, rules, caps, cancel. + +#![allow( + clippy::unwrap_used, + clippy::expect_used, + clippy::indexing_slicing, + clippy::panic +)] + +use std::fs; +use std::path::Path; +use std::sync::Mutex; +use std::time::{Duration, Instant}; + +use lutra_search::{ + CancelToken, FileMatches, MAX_FILE_SIZE, Overlay, Query, Search, SearchError, SearchEvent, + Summary, run, spawn, +}; +use proptest::prelude::*; + +/// Runs `query` over `root` and returns the files sorted by path. +fn find(root: &Path, query: &Query, overlay: &Overlay) -> (Vec, Summary) { + let search = Search::new(query).unwrap(); + let found = Mutex::new(Vec::new()); + let summary = run(root, &search, overlay, &CancelToken::new(), &|f| { + found.lock().unwrap().push(f) + }); + let mut files = found.into_inner().unwrap(); + files.sort_by(|a, b| a.path.cmp(&b.path)); + (files, summary) +} + +fn paths(files: &[FileMatches]) -> Vec<&str> { + files.iter().map(|f| f.path.as_str()).collect() +} + +fn put(root: &Path, rel: &str, text: &[u8]) { + let path = root.join(rel); + fs::create_dir_all(path.parent().unwrap()).unwrap(); + fs::write(path, text).unwrap(); +} + +#[test] +fn literal_case_and_whole_word() { + let dir = tempfile::tempdir().unwrap(); + put( + dir.path(), + "a.php", + b" 0); + assert_eq!(summary.files, tree.needle_files + 1); + assert!(!summary.truncated && !summary.cancelled); +} + +#[test] +fn skips_large_and_binary_files() { + let dir = tempfile::tempdir().unwrap(); + let mut big = b"needle\n".to_vec(); + big.resize(usize::try_from(MAX_FILE_SIZE).unwrap() + 1, b'a'); + put(dir.path(), "big.txt", &big); + let mut late_nul = b"needle\n".to_vec(); + late_nul.extend(b"filler\n".repeat(50_000)); + late_nul.push(0); + put(dir.path(), "late.bin", &late_nul); + put(dir.path(), "ok.txt", b"needle\n"); + let (files, _) = find(dir.path(), &Query::literal("needle"), &Overlay::new()); + assert_eq!(paths(&files), ["ok.txt"]); +} + +#[test] +fn overlay_replaces_the_disk_text() { + let dir = tempfile::tempdir().unwrap(); + put(dir.path(), "a.php", b"old text\n"); + put(dir.path(), "b.php", b"needle on disk\n"); + let mut overlay = Overlay::new(); + overlay.insert("a.php".into(), "line\n needle in buffer\n".into()); + overlay.insert("b.php".into(), "gone\n".into()); + overlay.insert("missing.php".into(), "needle\n".into()); + let (files, _) = find(dir.path(), &Query::literal("needle"), &overlay); + assert_eq!(paths(&files), ["a.php"]); + assert_eq!(files[0].matches[0].line, 1); + assert_eq!(files[0].matches[0].column, 2); +} + +#[test] +fn caps_the_number_of_matches() { + let dir = tempfile::tempdir().unwrap(); + for i in 0..20 { + put(dir.path(), &format!("f{i}.txt"), b"x x x x x\n"); + } + let mut q = Query::literal("x"); + q.max_results = 7; + let (files, summary) = find(dir.path(), &q, &Overlay::new()); + assert_eq!(summary.matches, 7); + assert!(summary.truncated); + assert_eq!(files.iter().map(|f| f.matches.len()).sum::(), 7); + assert_eq!(summary.files, files.len()); + + q.max_results = 100; + let (_, summary) = find(dir.path(), &q, &Overlay::new()); + assert_eq!(summary.matches, 100); + assert!( + !summary.truncated, + "exactly at the cap with nothing left out" + ); + q.max_results = 99; + assert!(find(dir.path(), &q, &Overlay::new()).1.truncated); +} + +#[test] +fn spawn_streams_and_cancels() { + let dir = tempfile::tempdir().unwrap(); + let tree = lutra_testkit::generate_search_tree(dir.path(), 300); + let search = Search::new(&Query::literal(lutra_testkit::NEEDLE)).unwrap(); + let rx = spawn( + dir.path().into(), + search.clone(), + Overlay::new(), + CancelToken::new(), + ) + .unwrap(); + let events: Vec = rx.iter().collect(); + let files = events + .iter() + .filter(|e| matches!(e, SearchEvent::File(_))) + .count(); + assert_eq!(files, tree.needle_files); + let Some(SearchEvent::Done(summary)) = events.last() else { + panic!("no summary last: {events:?}"); + }; + assert_eq!(summary.files, files); + + let cancel = CancelToken::new(); + cancel.cancel(); + let rx = spawn(dir.path().into(), search, Overlay::new(), cancel).unwrap(); + let events: Vec = rx.iter().collect(); + assert_eq!(events.len(), 1); + let SearchEvent::Done(summary) = &events[0] else { + panic!("{events:?}"); + }; + assert!(summary.cancelled); + assert_eq!(summary.matches, 0); +} + +/// Measures the first result and the total on a generated tree of 20,000 +/// files. Run in release: `cargo test -p lutra-search --release --test +/// search -- --ignored --nocapture measure`. +#[test] +#[ignore = "measurement; run explicitly in release"] +fn measure_20k_tree() { + let dir = tempfile::tempdir().unwrap(); + let made = Instant::now(); + let tree = lutra_testkit::generate_search_tree(dir.path(), 20_000); + println!("generated {} files in {:?}", tree.files, made.elapsed()); + for (label, pattern) in [ + ("common literal", "function"), + ("rare literal", lutra_testkit::NEEDLE), + ] { + for round in 0..3 { + let search = Search::new(&Query::literal(pattern)).unwrap(); + let started = Instant::now(); + let rx = spawn( + dir.path().into(), + search, + Overlay::new(), + CancelToken::new(), + ) + .unwrap(); + let mut first: Option = None; + let mut summary = None; + for event in rx { + match event { + SearchEvent::File(_) => { + first.get_or_insert_with(|| started.elapsed()); + } + SearchEvent::Done(s) => summary = Some(s), + } + } + let s = summary.unwrap(); + println!( + "{label} round {round}: first {:?}, total {:?}, files {}, matches {}, truncated {}", + first.unwrap(), + s.elapsed, + s.files, + s.matches, + s.truncated + ); + } + } + let mut q = Query::literal("function"); + q.max_results = lutra_search::MAX_RESULTS_LIMIT; + let search = Search::new(&q).unwrap(); + let s = run( + dir.path(), + &search, + &Overlay::new(), + &CancelToken::new(), + &|_| {}, + ); + println!( + "common literal, cap 100k: total {:?}, files {}, matches {}, truncated {}", + s.elapsed, s.files, s.matches, s.truncated + ); +} + +proptest! { + #![proptest_config(ProptestConfig::with_cases(64))] + + /// Random patterns, flags and file contents never panic, and every + /// reported match lies inside its preview. + #[test] + fn random_queries_never_panic( + pattern in "\\PC{0,12}", + regex in any::(), + case_sensitive in any::(), + whole_word in any::(), + content in proptest::collection::vec(any::(), 0..400), + text in "\\PC{0,200}", + ) { + let dir = tempfile::tempdir().unwrap(); + fs::write(dir.path().join("bytes.txt"), &content).unwrap(); + fs::write(dir.path().join("text.php"), &text).unwrap(); + let query = Query { + pattern, + regex, + case_sensitive, + whole_word, + include: vec![], + exclude: vec![], + max_results: 50, + }; + let Ok(search) = Search::new(&query) else { + return Ok(()); + }; + let found = Mutex::new(Vec::new()); + let summary = run(dir.path(), &search, &Overlay::new(), &CancelToken::new(), &|f| { + found.lock().unwrap().push(f) + }); + let found = found.into_inner().unwrap(); + prop_assert!(summary.matches <= 50); + for file in found { + for m in file.matches { + prop_assert!(m.length > 0); + let units = m.preview.encode_utf16().count(); + prop_assert!(m.preview_column + m.length <= units || m.preview_column <= units); + } + } + } +} diff --git a/crates/testkit/src/lib.rs b/crates/testkit/src/lib.rs index 3e117e6..823ab22 100644 --- a/crates/testkit/src/lib.rs +++ b/crates/testkit/src/lib.rs @@ -10,6 +10,8 @@ clippy::indexing_slicing )] +pub mod tree; + use std::path::{Path, PathBuf}; use std::sync::Mutex; @@ -17,6 +19,8 @@ use lutra_control::{Backend, EventSink, Subscribers}; use lutra_engine::{CommandInfo, Effect, Engine, EngineConfig, EngineError}; use serde_json::Value; +pub use tree::{NEEDLE, SearchTree, generate_search_tree}; + /// Engine directories inside `root` (`root/config`, `root/data`), for /// [`Engine::with_config`] and [`EngineBackend::with_config`] in tests. pub fn engine_config(root: &Path) -> EngineConfig { diff --git a/crates/testkit/src/tree.rs b/crates/testkit/src/tree.rs new file mode 100644 index 0000000..d7e359f --- /dev/null +++ b/crates/testkit/src/tree.rs @@ -0,0 +1,151 @@ +//! Generated project trees for search tests and measurements. +//! +//! [`generate_search_tree`] writes a deterministic PHP project of any size: +//! project sources, templates, scripts and docs, a gitignored `var/`, a +//! gitignored Composer `vendor/` (searched as library code), an excluded +//! `node_modules/` and a binary file. Every [`NEEDLE_EVERY`]th file of each +//! group contains [`NEEDLE`] once; [`SearchTree`] reports how many files of +//! each group have it, so tests know the expected result without searching +//! twice. + +use std::fs; +use std::path::Path; + +/// A token that occurs only where the generator puts it. +pub const NEEDLE: &str = "lutraNeedle"; + +/// Every this many files of a group contains [`NEEDLE`]. +pub const NEEDLE_EVERY: usize = 50; + +/// What [`generate_search_tree`] wrote. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct SearchTree { + /// Files written in total, including `.gitignore` and the binary file. + pub files: usize, + /// Searchable files (sources, templates, scripts, docs, vendor) that + /// contain [`NEEDLE`]. + pub needle_files: usize, + /// Of those, files under `vendor/`. + pub needle_vendor: usize, + /// Files containing [`NEEDLE`] that a search must skip (ignored, + /// excluded or binary). + pub needle_skipped: usize, +} + +/// Writes a project of about `files` files below `root`. +/// +/// Shares: 70 % PHP sources in `src/`, 8 % templates, 5 % JavaScript, 5 % +/// Markdown, 6 % ignored PHP in `var/cache/`, 6 % PHP in `vendor/`, plus a +/// few files in `node_modules/` and `bin/data.bin` (a NUL byte, then the +/// needle). Every PHP file contains `function` on several lines, the common +/// literal for measurements. +/// +/// # Panics +/// If a file cannot be written (test helper). +pub fn generate_search_tree(root: &Path, files: usize) -> SearchTree { + let mut tree = SearchTree::default(); + write(root, ".gitignore", "/var/\n/vendor/\n*.log\n", &mut tree); + let groups: [(&str, usize, Kind); 6] = [ + ("src", 70, Kind::Php), + ("templates", 8, Kind::Twig), + ("assets", 5, Kind::Js), + ("docs", 5, Kind::Markdown), + ("var/cache", 6, Kind::Php), + ("vendor/acme/toolkit/src", 6, Kind::Php), + ]; + for (dir, share, kind) in groups { + let count = files.saturating_mul(share) / 100; + for i in 0..count { + let needle = i % NEEDLE_EVERY == 0; + let rel = format!("{dir}/Module{:03}/{}", i / 100, kind.file_name(i)); + write(root, &rel, &kind.content(i, needle), &mut tree); + if !needle { + continue; + } + if dir == "var/cache" { + tree.needle_skipped += 1; + } else { + tree.needle_files += 1; + if dir.starts_with("vendor") { + tree.needle_vendor += 1; + } + } + } + } + for i in 0..3 { + let rel = format!("node_modules/pkg{i}/index.js"); + write(root, &rel, &Kind::Js.content(i, true), &mut tree); + tree.needle_skipped += 1; + } + write(root, "debug.log", &format!("{NEEDLE}\n"), &mut tree); + tree.needle_skipped += 1; + let mut binary = b"\x89BIN\x00\x01\x02".to_vec(); + binary.extend_from_slice(NEEDLE.as_bytes()); + fs::create_dir_all(root.join("bin")).unwrap(); + fs::write(root.join("bin/data.bin"), binary).unwrap(); + tree.files += 1; + tree.needle_skipped += 1; + tree +} + +/// Writes `text` to `root/rel`, creating directories, and counts it. +fn write(root: &Path, rel: &str, text: &str, tree: &mut SearchTree) { + let path = root.join(rel); + if let Some(parent) = path.parent() { + fs::create_dir_all(parent).unwrap(); + } + fs::write(path, text).unwrap(); + tree.files += 1; +} + +/// The kind of a generated file. +#[derive(Clone, Copy)] +enum Kind { + /// A PHP class. + Php, + /// A Twig template. + Twig, + /// A JavaScript module. + Js, + /// A Markdown page. + Markdown, +} + +impl Kind { + /// File name of the `i`th file. + fn file_name(self, i: usize) -> String { + match self { + Kind::Php => format!("Service{i}.php"), + Kind::Twig => format!("page{i}.html.twig"), + Kind::Js => format!("widget{i}.js"), + Kind::Markdown => format!("note{i}.md"), + } + } + + /// Content of the `i`th file, with [`NEEDLE`] once if `needle`. + fn content(self, i: usize, needle: bool) -> String { + let marker = if needle { NEEDLE } else { "plain" }; + match self { + Kind::Php => { + let mut s = format!( + "items[$key])) {{\n return $this->items[$key];\n }}\n return null;\n }}\n\n" + )); + } + s.push_str(&format!(" public const MARKER = '{marker}';\n}}\n")); + s + } + Kind::Twig => format!( + "{{% extends 'base.html.twig' %}}\n{{% block body %}}\n

Page {i}

\n

{{{{ content }}}}

\n{{% endblock %}}\n" + ), + Kind::Js => format!( + "export function widget{i}(el) {{\n el.dataset.kind = '{marker}';\n return el;\n}}\n" + ), + Kind::Markdown => format!("# Note {i}\n\nSome text about {marker} and more text.\n"), + } + } +} diff --git a/docs/architecture.md b/docs/architecture.md index 49138d6..ed739c7 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -1,6 +1,6 @@ # Architecture -> **Status:** M0 decisions recorded, M1 workspace, desktop bridge, tabs and editor, message catalog, engine construction, settings, themes, the recovery journal, notifications, UI commands, the palette, the settings dialog, the app icon and the syntax layer documented ยท **Last updated:** 2026-10-16 +> **Status:** M0 decisions recorded, M1 workspace, desktop bridge, tabs and editor, message catalog, engine construction, settings, themes, the recovery journal, notifications, UI commands, the palette, the settings dialog, the app icon and the syntax layer documented, plus the search backend ยท **Last updated:** 2026-10-16 ## Overview @@ -79,7 +79,7 @@ Theme tokens are defined in Rust (`lutra_engine::theme`) and rendered as CSS cus ### Events -`lutra_engine::Event`, serialised as `{"kind": "", ...}`: `projectOpened {root, name}`, `projectClosed`, `treeChanged {dir}`, `bufferOpened {id, path}`, `bufferFocused {id}` (M1.13), `bufferChanged {id, version}`, `dirtyChanged {id, dirty}`, `bufferClosed {id}`, `fileChangedOnDisk {path, id, removed}`, `watchFailed {message}`, `recoveryAvailable {entries}` (M1.7, see Recovery journal), `uiCommand {id, params}` (M1.8, see UI commands), `recoveryResolved {paths}` (M1.15, see Recovery journal). Events say what changed; clients fetch data with `project.children` and `buffer.get`. `file.open` on a file whose buffer is already open returns that buffer and emits `bufferFocused {id}` instead of `bufferOpened` (M1.13): a separate event, so that `bufferOpened` keeps meaning "a new buffer exists" (the host's `engine_ready` replay and other clients rely on that) while every client, including the editor, still learns that the user asked to see that buffer and activates its tab. +`lutra_engine::Event`, serialised as `{"kind": "", ...}`: `projectOpened {root, name}`, `projectClosed`, `treeChanged {dir}`, `bufferOpened {id, path}`, `bufferFocused {id}` (M1.13), `bufferChanged {id, version}`, `dirtyChanged {id, dirty}`, `bufferClosed {id}`, `fileChangedOnDisk {path, id, removed}`, `watchFailed {message}`, `recoveryAvailable {entries}` (M1.7, see Recovery journal), `uiCommand {id, params}` (M1.8, see UI commands), `recoveryResolved {paths}` (M1.15, see Recovery journal), `searchResults {searchId, path, matches}` and `searchDone {searchId, files, matches, truncated, cancelled, elapsedMs}` (M2.3, see Search). Events say what changed; clients fetch data with `project.children` and `buffer.get`. `file.open` on a file whose buffer is already open returns that buffer and emits `bufferFocused {id}` instead of `bufferOpened` (M1.13): a separate event, so that `bufferOpened` keeps meaning "a new buffer exists" (the host's `engine_ready` replay and other clients rely on that) while every client, including the editor, still learns that the user asked to see that buffer and activates its tab. Changes on disk reach the engine through `Engine::pump_vfs()`, which hosts call regularly (the desktop's pump thread every 100 ms; tests call it directly). It maps VFS events to `treeChanged` for the parent directories, reloads a clean buffer whose file changed (unless the file already matches, as after our own save) with `bufferChanged`, and reports `fileChangedOnDisk` for a dirty buffer, which keeps its text, or for a removed file. At most 10,000 VFS events are handled per pump. @@ -207,6 +207,17 @@ Watched directories: the VFS watcher is non-recursive, so `project.children` wat - **Limits:** texts over 8 MiB (`MAX_PARSE_BYTES`) are not parsed (`truncated`, empty outline and folding). The grammar's identifier rule covers the Basic Multilingual Plane only: PHP accepts any byte โ‰ฅ 0x80 in names, so an identifier with an emoji is an error node in tree-sitter. - **Timing** (release, Windows 11, MSVC `/O2`, generated 1 MB file, best of 5, other builds running in parallel, measured): full parse 364โ€“553 ms, reparse after one inserted character 7โ€“11 ms, outline 70โ€“133 ms, folding 61โ€“73 ms (143 ms before skipping single-line nodes). The full parse misses the 50 ms target by about 8ร—: tree-sitter-php runs at roughly 2โ€“3 MB/s here. Consequences: parse off the engine lock and debounce (M2.7), parse files in parallel for the index (M2.5); `cargo test --release -p lutra-syntax --test perf -- --ignored --nocapture` repeats the measurement. +### Search (M2.3) + +- `crates/search` (L2) is the library: `Search::new(&Query)` compiles a literal or regex (`grep-regex`; empty, invalid, oversized or newline-matching patterns and bad globs fail up front), `run` walks the project with the `ignore` crate's parallel walker (at most 8 threads) and searches each file with `grep-searcher`; `spawn` runs it on a thread of its own and delivers `SearchEvent::File` per file with matches, then one `SearchEvent::Done`. A `CancelToken` stops it between files and lines. +- Which files: the same rules as `lutra-vfs` (`.gitignore`, `.ignore`, `.git/info/exclude`, global excludes, applied without a git repository; ignore files above the root are not read). `.git` and `node_modules` are skipped, the root's `vendor/` is searched as library code even when an ignore file lists it (it is walked as a second root, so its own nested ignore files still apply), dot files are searched, symbolic links are not followed. Include globs keep matching files, exclude globs drop files and whole directories; a glob without `/` matches the name, one with `/` the project-relative path. Files over 8 MiB, binary files (a NUL byte) and unreadable files are skipped silently. The library does not depend on `lutra-vfs`: the walker is configured to the same rules instead, because the VFS reads rules per directory listing and the walker needs them per walk. +- Results: positions are 0-based, columns and lengths in UTF-16 code units, lines end at ` +` (a preceding ` +` is not part of the line; a lone ` +` is not a line break here, unlike in `lutra-text`). Invalid UTF-8 is replaced before matching. The preview is the line, or a window of 300 characters starting up to 80 characters before the match. Files arrive in no fixed order. The cap (`maxResults`, default 10,000, at most 100,000) counts matches; `truncated` is set only when a match was left out. +- Engine: `search.find {query, regex?, caseSensitive?, wholeWord?, include?, exclude?, maxResults?}` (disabled without a project) snapshots the text of every dirty buffer as an overlay (searched instead of the file; only for files the walk visits), starts the search and returns `{searchId}` at once. Searches run outside the engine lock. Each search's results wait in its own channel in `State::searches`; `Engine::pump_vfs` drains them (`cmd::search::pump`) into `searchResults` events (one per file) and a final `searchDone`. Hosts already publish the pump's events, so the desktop (pump every 100 ms) and the control channel get them without new plumbing; there are at most 10 batches per second and at most 500 `searchResults` per pump (`MAX_RESULTS_PER_PUMP`), which keeps a control subscriber's queue of 4096 events from overflowing; the rest wait for the next pump. `search.cancel {searchId}` returns `{cancelled}` (false once the search ended) and the search still ends with `searchDone {cancelled: true}`. Starting a search does not cancel older ones. A search whose project is no longer open is cancelled by the next pump, its remaining results are dropped and it ends with `cancelled: true`. +- Measured (release, Windows 11, 16 threads, generated 20,006-file tree from `lutra_testkit::generate_search_tree`, other builds running): common literal `function` first result 10โ€“43 ms, capped at 10,000 matches after 112โ€“209 ms; a rare literal (full walk, 376 files) first result 17โ€“22 ms, total 0.66โ€“1.43 s (the first round cold); `function` with the 100,000 cap 0.53 s. Through the engine add up to one pump interval (100 ms) before results reach clients. Live check over the control channel against the running app (debug build, 5,900-file project with a dirty buffer): `search.find` answered in 4.5 ms, first `searchResults` line after 55 ms, `searchDone` after 386 ms; vendor hits present, `var/`, `node_modules` and the binary file absent; the dirty buffer's text was found; `search.cancel` ended a search with `cancelled: true`. + ## Decisions (M0) | Question | Decision | Why | @@ -224,6 +235,7 @@ Watched directories: the VFS watcher is non-recursive, so `project.children` wat | Date | Change | Summary | |---|---|---| +| 2026-10-16 | minor | M2.3: find in files backend (`crates/search`, `search.find`/`search.cancel`, `searchResults`/`searchDone` via the pump) | | 2026-10-16 | minor | M2.1: syntax layer (tree-sitter PHP parser, incremental reparse, outline, folding, size cap, timing) | | 2026-10-16 | minor | M1.15: modal rule (one modal at a time, restore dialog queued), `recoveryResolved` event | | 2026-10-16 | minor | M1.9b: settings dialog (store, search, controls by type, reset, live updates, font-size token override) | diff --git a/docs/gaps.md b/docs/gaps.md index 81ef751..b1a0682 100644 --- a/docs/gaps.md +++ b/docs/gaps.md @@ -31,6 +31,8 @@ G11 progress: M1.7 added the crash-recovery journal: unsaved buffers are written G14 progress: M1.11 moved every UI string in `ui/src` into the English catalog `ui/src/i18n/en.json` behind `t(key, params?)`, enforced by `vue/no-bare-strings-in-template` and a catalog test. Messages the engine sends (`EngineError.message`, the slogan from `app.about`) are still English text; they need message keys with params before a second language can cover them. +G5 progress: M2.3 landed the find-in-files backend: `crates/search` (parallel walk with the project's ignore rules, literal or regex, case, whole word, globs, binary and size skips, match cap, cancel) and the engine commands `search.find` / `search.cancel` with streamed `searchResults` and `searchDone` events; dirty buffers are searched in their buffer text. First result on a generated 20k-file tree 10โ€“43 ms (release, measured). Open in G5: the Find tool window (M2.4), replace in files, the symbol index and go-to (M2.5, M2.6). + Known smaller items from M0: - The CI workflow has never run: the repository has no remote until the owner decides on hosting. @@ -41,6 +43,7 @@ Known smaller items from M0: | Date | Change | Summary | |---|---|---| +| 2026-10-16 | minor | G5: find-in-files backend (M2.3) | | 2026-10-16 | minor | G4 note: tree-sitter syntax library, outline and folding (M2.1) | | 2026-10-16 | minor | G3: settings dialog, theme picker (M1.9b) | | 2026-10-16 | minor | G3: command palette, global keymap, UI commands (M1.8) | diff --git a/docs/target-app-parity.md b/docs/target-app-parity.md index ec051b2..703824f 100644 --- a/docs/target-app-parity.md +++ b/docs/target-app-parity.md @@ -15,7 +15,7 @@ Hours are the estimated agent wall-clock hours for one agent to reach behaviour | Editor core | 12 | 9% (est.) | syntax highlighting (M2), code folding, soft wrap and other editor settings, preview tab, pinned/split/reordered tabs, close others/all, reopen closed tab, tab limit, conflict UI for files changed on disk, autosave, local history, encodings other than UTF-8, large-file mode, column selection and multi-caret checked against the docs; recovery journal not yet checked in the running app (gaming mode) | ~106 (est.) | M2.1: `crates/syntax` parses PHP with tree-sitter (incremental reparse from `lutra_text` edits) and computes folding ranges (brace blocks, arrays, comments, heredocs, `use` runs); library only, nothing visible in the editor yet, so the score is unchanged. Tests: outline/folding snapshots for five PHP 8.3 fixtures, proptests (random bytes and token soups never panic, reparse equals a fresh parse, undo round trip). 1 MB parse 364โ€“553 ms, reparse after one keystroke 7โ€“11 ms (release, measured, machine busy). M1.13: opening a file that is already open switches to its tab (`bufferFocused`; engine test, Vitest, control-channel test; live check pending (gaming mode)); files in folders the tree never expanded still reload or report changes on disk, because the parent directory of every open buffer stays watched (engine tests). M1.7: crash-recovery journal (`crates/engine/src/journal.rs`, `cmd/recovery.rs`): dirty buffers are written atomically to `/recovery//` about 1 s after the last edit (at least every 2 s while typing), never per keystroke; removed on save, close and discard; `recoveryAvailable` after `project.open`, `recovery.list/restore/discard`; restored buffers are dirty and keep line ending and BOM; corrupt entries are moved aside with a warning. Restore dialog (`ui/src/components/recovery/`: Restore all / selected / Discard, time, changed-on-disk hint). Tests: engine dropped mid-edit and a new engine restores the exact CRLF + BOM text, corrupt entries, removal paths, a property test of random edit sequences โ†’ crash โ†’ restore equals the buffer (`crates/engine/tests/recovery.rs`, 10 tests), the same flow headless over the control channel (`crates/control/tests/recovery.rs`), 10 Vitest tests for the dialog. 1 MiB entry write: 8.5 ms median (release, measured). Graded against `plan/phpstorm/04-recovery.md`. M1.6: tabs and a CodeMirror 6 editor in the app (`ui/src/components/editor/`, `ui/src/stores/editor.ts`): open from the project tree or engine events, activate, close by button, middle click or `Mod+W`, dirty dot, Save / Don't Save / Cancel on closing a dirty tab, `Mod+S` / `Mod+Alt+S` from engine command specs, non-blocking notice for a dirty file changed or deleted on disk; local edits go to `buffer.edit` with version-based resync, CRLF files keep `\r\n`; undo/redo, multiple selections, search and bracket matching from CodeMirror's basic setup. Vitest: a burst of 400 keystrokes with stale versions and remote edits loses none, a CRLF burst with deletions converges, store and component tests against a fake engine (`ui/tests/editor/`, 22 tests). Checked in the app on Windows 11 with `--control` and `ui.screenshot`: typing via the UI and via `buffer.edit`, save to disk, close question, middle click, disk-change notice. 1 MB file (16,385 lines): engine `file.open` 80 ms over the control channel (debug build, measured), editor state 1โ€“6 ms and first rendered view 4โ€“11 ms in Chromium (measured), โ‰ˆ 90 ms in total (estimated, budget 150 ms). Graded against `plan/phpstorm/03-editor-tabs.md`. `crates/text`: rope buffer, positions, edits (M1.1, unit tests + proptests). M1.3: engine buffers with versions and dirty state, `file.open`, `buffer.get`, `buffer.edit` (UTF-16 change sets, `stale_version`), `file.save`, `file.saveAll`, `buffer.close` with unsaved-changes guard, line ending and BOM kept on save, clean buffers reload on external change; 14 integration tests on temp dirs (`crates/engine/tests/workspace.rs`) | | Project view, VFS | 6 | 8% (est.) | views list (project / scopes), toolbar (Select Opened File, Expand/Collapse All), sort and display options, show/hide excluded, distinct icons for library and excluded folders, file colours, speed search, preview, file operations | ~37 (est.) | `lutra-vfs` (M1.2): lazy sorted listing, ignore rules (.gitignore, .ignore, info/exclude, global excludes, Lutra defaults), native file watching via `notify` with a polling fallback, capped UTF-8 reads, atomic writes; 34 tests on temp dirs, watcher tests run on both backends (`crates/vfs/tests`). M1.5: project tree UI (lazy listing, keyboard navigation, dimmed ignored entries, event refresh) in `ui/src/components/project/`, Vitest against a mocked engine. M1.3: `project.open`, `project.close`, `project.children` (watches listed directories, capped at 256 native / 32 polling, least recently listed dropped), `treeChanged` events from disk, `events.subscribe` on the control channel with a TCP test (`crates/control/tests/events.rs`). M1.4: the tree runs against the engine in the app: folder picker (`project.openDialog`) and `lutra-desktop --control PORT PATH` open a project, the tree lists it lazily with ignored (`var/` from `.gitignore`) and excluded (`node_modules`) entries dimmed, a file created on disk appears in an expanded folder within ~100 ms; verified with `ui.screenshot` on Windows 11 and Vitest (`ui/tests/project/`, `ui/tests/ipcEvents.test.ts`). M1.13: a directory listed only partly (more than 100,000 entries) ends with a "Listing truncated" note row (Vitest). M1.12: a folder that cannot be opened from the picker or the command line (including `unsaved_changes` when switching projects) and stopped file watching show as toasts instead of stderr (`apps/lutra-desktop/src/host.rs`, `ui/src/stores/notifications.ts`, `ui/tests/notifications/`, 19 Vitest tests; live check pending, gaming mode); score unchanged. Graded against `plan/phpstorm/02-project-view.md`: tool window, folder-first tree, dimmed ignored/excluded, keyboard navigation and live refresh exist; most toolbar, view and display options do not | | Navigation | 10 | 0% | everything (M1.8: a command palette over commands only, no go-to-file/class/symbol yet; score unchanged) | ~120 (est.) | M2.1: file outline in `crates/syntax` (namespaces braced and unbraced, classes, interfaces, traits, enums with cases, functions, methods, properties incl. promoted ones, constants; UTF-16 ranges and short signatures), the base for the Structure window (M2.7) and the symbol index (M2.5); not reachable from the UI yet, score unchanged. M1.8: `ui/src/components/palette/`, `ui/src/commands/` (fuzzy ranking, recent commands, global keymap), Vitest `ui/tests/palette/`, `ui/tests/commands/` | -| Search and replace | 6 | 0% | everything | ~80 (est.) | โ€” | +| Search and replace | 6 | 0% (est.) | everything a user sees: Find in Files tool window (M2.4), replace in files, scopes, file masks UI, search history, find in path from the tree, structural search; the backend below has no UI yet, so the score stays 0% | ~76 (est.) | M2.3 backend: `crates/search` (parallel `ignore` walk + `grep-searcher`: literal/regex, case, whole word, include/exclude globs, project ignore rules with `vendor/` searched and `node_modules` skipped, binary and > 8 MiB files skipped, match cap, cancel) and engine `search.find` / `search.cancel` with `searchResults` / `searchDone` events through the pump; dirty buffers are searched in their buffer text. Tests: 9 library tests incl. a property test of random queries and file bytes (`crates/search/tests/search.rs`), 3 preview/column unit tests, 4 engine tests (`crates/engine/tests/search.rs`), a TCP control-channel test (find โ†’ results โ†’ done, invalid regex, cancel; `crates/control/tests/search.rs`). 20k-file generated tree, release: first result 10โ€“43 ms, full walk 0.66โ€“1.43 s (measured); live over the control channel against the running app: first result 55 ms, done 386 ms on a 5,900-file project (debug build, measured) | | Code completion | 10 | 0% | everything | ~250 (est.) | โ€” | | Inspections and quick-fixes | 8 | 0% | everything | ~300 (est.) | โ€” | | Refactoring | 7 | 0% | everything | ~250 (est.) | โ€” | @@ -63,6 +63,7 @@ The project tree is the first user-visible feature (Project view 8%, estimated); | Date | Change | Summary | |---|---|---| +| 2026-10-16 | minor | M2.3: Search and replace evidence (find-in-files backend, no UI), score unchanged at 0%, hours 80 โ†’ 76 | | 2026-10-16 | minor | M2.1: Editor core and Navigation evidence (tree-sitter parse, outline, folding library), scores unchanged | | 2026-10-16 | minor | M1.9b: Settings row 4% โ†’ 7% (est.), settings dialog; M1 done | | 2026-10-16 | minor | M1.8: Navigation and Settings/keymaps evidence (palette, global keymap), scores unchanged |