M2.3: find in files backend

crates/search: parallel ignore walk + grep-searcher with the project's
ignore rules (vendor searched as library, node_modules and .git skipped),
literal or regex, case, whole word, include/exclude globs, binary and
> 8 MiB skips, match cap, cancel token, buffer overlay.

Engine: search.find returns {searchId} at once and runs off the lock;
pump_vfs drains results into searchResults (one per file, at most 500
per pump) and searchDone; search.cancel. Testkit: generated search tree.
Tests: library unit and property tests, engine tests, TCP control test.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
danny 2026-10-16 22:03:17 +02:00
parent a6b2fd1300
commit 172693c8dd
21 changed files with 2231 additions and 5 deletions

View file

@ -12,6 +12,7 @@ publish.workspace = true
layer = 4
[dependencies]
lutra-search.workspace = true
lutra-text.workspace = true
lutra-vfs.workspace = true
serde = { version = "1.0.229", features = ["derive"] }
@ -22,5 +23,6 @@ thiserror = "2.0.21"
workspace = true
[dev-dependencies]
lutra-testkit.workspace = true
proptest.workspace = true
tempfile.workspace = true

View file

@ -6,6 +6,7 @@ mod file;
mod params;
mod project;
mod recovery;
pub mod search;
mod settings;
pub mod theme;
pub mod ui;
@ -39,5 +40,7 @@ pub fn register_builtins(registry: &mut Registry) -> Result<(), EngineError> {
registry.register(Box::new(theme::List))?;
registry.register(Box::new(theme::SystemPreference))?;
registry.register(Box::new(ui::RegisterCommands))?;
registry.register(Box::new(search::Find))?;
registry.register(Box::new(search::Cancel))?;
Ok(())
}

View file

@ -0,0 +1,304 @@
//! Find in files: `search.find` and `search.cancel`.
//!
//! `search.find` compiles the query, takes a snapshot of the dirty buffers
//! and starts the search on its own threads ([`lutra_search::spawn`]), then
//! returns `{searchId}` at once; the engine lock is never held while files
//! are read. Results wait in a channel per search until
//! [`crate::Engine::pump_vfs`] calls [`pump`], which turns them into
//! [`Event::SearchResults`] (one per file) and a final
//! [`Event::SearchDone`]. Hosts already publish the pump's events, so
//! results reach the window and control-channel subscribers in the
//! engine's event order, at most one batch per pump (every 100 ms on the
//! desktop) and at most [`MAX_RESULTS_PER_PUMP`] files per batch, so a
//! subscriber's queue never overflows. Searches whose project is no longer
//! open are cancelled by the next pump and end with `cancelled: true`.
use std::collections::BTreeMap;
use std::path::{Component, PathBuf};
use std::sync::mpsc::{Receiver, TryRecvError};
use std::time::Instant;
use lutra_search::{CancelToken, Overlay, Query, Search, SearchEvent};
use serde::Deserialize;
use serde_json::{Value, json};
use crate::cmd::params::parse;
use crate::{Command, CommandSpec, EngineError, Event, Outcome, SearchMatch, State};
/// Most `searchResults` events one pump emits, over all searches; the rest
/// wait for the next pump.
pub const MAX_RESULTS_PER_PUMP: usize = 500;
/// Most include or exclude globs one query may carry.
pub const MAX_GLOBS: usize = 256;
/// The running searches.
#[derive(Debug, Default)]
pub struct Searches {
/// The id of the last search started; ids are never reused.
last_id: u64,
/// Searches whose `searchDone` has not been emitted yet, by id.
running: BTreeMap<u64, Running>,
}
impl Searches {
/// Number of searches whose `searchDone` is still to come.
pub fn running(&self) -> usize {
self.running.len()
}
}
/// One search in flight.
#[derive(Debug)]
struct Running {
/// The project root the search walks.
root: PathBuf,
/// Stops it.
cancel: CancelToken,
/// Its results and summary.
events: Receiver<SearchEvent>,
/// When it started, for a summary the search thread never sent.
started: Instant,
/// Files emitted so far.
files: usize,
/// Matches emitted so far.
matches: usize,
}
/// Params of `search.find`.
#[derive(Deserialize)]
#[serde(deny_unknown_fields, rename_all = "camelCase")]
struct FindParams {
/// The text or regular expression.
query: String,
/// `query` is a regular expression.
#[serde(default)]
regex: bool,
/// Match letter case exactly.
#[serde(default)]
case_sensitive: bool,
/// Whole words only.
#[serde(default)]
whole_word: bool,
/// Only files matching one of these globs.
#[serde(default)]
include: Vec<String>,
/// Skip files and directories matching one of these globs.
#[serde(default)]
exclude: Vec<String>,
/// Cap on matches.
max_results: Option<usize>,
}
/// `search.find`: starts a search in the project.
pub struct Find;
impl Command for Find {
fn spec(&self) -> &CommandSpec {
&CommandSpec {
id: "search.find",
label: "Find in Files",
menu_path: None,
default_shortcut: None,
params: "{ query, regex?, caseSensitive?, wholeWord?, include?: [glob], exclude?: [glob], maxResults? (default 10000, at most 100000) }; returns { searchId } at once, then events searchResults { searchId, path, matches: [{ line, column, length, preview, previewColumn }] } per file and searchDone { searchId, files, matches, truncated, cancelled, elapsedMs }; invalid_params for an empty query, a bad regex or glob",
}
}
fn enabled(&self, state: &State) -> bool {
state.has_project()
}
fn run(&self, state: &mut State, params: Value) -> Result<Outcome, EngineError> {
let id = self.spec().id;
let p: FindParams = parse(id, params)?;
if p.include.len() > MAX_GLOBS || p.exclude.len() > MAX_GLOBS {
return Err(EngineError::invalid_params(
id,
format!("at most {MAX_GLOBS} include and {MAX_GLOBS} exclude globs"),
));
}
let query = Query {
pattern: p.query,
regex: p.regex,
case_sensitive: p.case_sensitive,
whole_word: p.whole_word,
include: p.include,
exclude: p.exclude,
max_results: p.max_results.unwrap_or(lutra_search::DEFAULT_MAX_RESULTS),
};
let search =
Search::new(&query).map_err(|e| EngineError::invalid_params(id, e.to_string()))?;
let root = state
.project
.as_ref()
.ok_or_else(|| EngineError::Disabled(id.to_string()))?
.project
.root()
.to_path_buf();
let cancel = CancelToken::new();
let events = lutra_search::spawn(root.clone(), search, overlay(state), cancel.clone())
.map_err(|e| EngineError::Failed(format!("cannot start the search: {e}")))?;
let searches = &mut state.searches;
searches.last_id = searches.last_id.saturating_add(1);
let search_id = searches.last_id;
searches.running.insert(
search_id,
Running {
root,
cancel,
events,
started: Instant::now(),
files: 0,
matches: 0,
},
);
Ok(Outcome::value(json!({ "searchId": search_id })))
}
}
/// Params of `search.cancel`.
#[derive(Deserialize)]
#[serde(deny_unknown_fields, rename_all = "camelCase")]
struct CancelParams {
/// The search to stop.
search_id: u64,
}
/// `search.cancel`: stops a running search.
pub struct Cancel;
impl Command for Cancel {
fn spec(&self) -> &CommandSpec {
&CommandSpec {
id: "search.cancel",
label: "Cancel Search",
menu_path: None,
default_shortcut: None,
params: "{ searchId }; returns { cancelled }: false if the search already ended or never existed; a cancelled search still ends with searchDone (cancelled: true)",
}
}
fn run(&self, state: &mut State, params: Value) -> Result<Outcome, EngineError> {
let p: CancelParams = parse(self.spec().id, params)?;
let running = state.searches.running.get(&p.search_id);
if let Some(running) = running {
running.cancel.cancel();
}
Ok(Outcome::value(json!({ "cancelled": running.is_some() })))
}
}
/// The text of every dirty buffer by project-relative path, searched
/// instead of the file on disk. A path that is not plain Unicode cannot
/// match a walked file and is left out.
fn overlay(state: &State) -> Overlay {
state
.buffers
.dirty_ids()
.into_iter()
.filter_map(|id| state.buffers.get(id))
.filter_map(|b| Some((slash_path(&b.path)?, b.buffer.text())))
.collect()
}
/// A project-relative path with `/` separators.
fn slash_path(path: &std::path::Path) -> Option<String> {
let parts: Option<Vec<&str>> = path
.components()
.map(|c| match c {
Component::Normal(name) => name.to_str(),
_ => None,
})
.collect();
Some(parts?.join("/"))
}
/// Moves finished results of running searches into events: at most
/// [`MAX_RESULTS_PER_PUMP`] `searchResults`, and `searchDone` for every
/// search that ended. Searches of a project that is no longer open are
/// cancelled; their remaining results are dropped. A search thread that
/// vanished without a summary ends with `cancelled: true` and the counts
/// emitted so far.
pub fn pump(state: &mut State) {
let root = state
.project
.as_ref()
.map(|w| w.project.root().to_path_buf());
let mut budget = MAX_RESULTS_PER_PUMP;
let mut events = Vec::new();
let mut finished = Vec::new();
for (&search_id, running) in state.searches.running.iter_mut() {
let stale = root.as_ref() != Some(&running.root);
if stale {
running.cancel.cancel();
}
loop {
if budget == 0 && !stale {
break;
}
match running.events.try_recv() {
Ok(SearchEvent::File(file)) => {
if stale {
continue;
}
budget = budget.saturating_sub(1);
running.files = running.files.saturating_add(1);
running.matches = running.matches.saturating_add(file.matches.len());
events.push(Event::SearchResults {
search_id,
path: file.path,
matches: file.matches.into_iter().map(SearchMatch::from).collect(),
});
}
Ok(SearchEvent::Done(summary)) => {
events.push(Event::SearchDone {
search_id,
files: summary.files,
matches: summary.matches,
truncated: summary.truncated,
cancelled: summary.cancelled || stale,
elapsed_ms: millis(summary.elapsed),
});
finished.push(search_id);
break;
}
Err(TryRecvError::Empty) => break,
Err(TryRecvError::Disconnected) => {
events.push(Event::SearchDone {
search_id,
files: running.files,
matches: running.matches,
truncated: false,
cancelled: true,
elapsed_ms: millis(running.started.elapsed()),
});
finished.push(search_id);
break;
}
}
}
}
for search_id in finished {
state.searches.running.remove(&search_id);
}
for event in events {
state.emit(event);
}
}
/// Whole milliseconds of `elapsed`, saturating.
fn millis(elapsed: std::time::Duration) -> u64 {
u64::try_from(elapsed.as_millis()).unwrap_or(u64::MAX)
}
impl From<lutra_search::Match> for SearchMatch {
fn from(m: lutra_search::Match) -> Self {
Self {
line: m.line,
column: m.column,
length: m.length,
preview: m.preview,
preview_column: m.preview_column,
}
}
}

View file

@ -130,7 +130,8 @@ impl Engine {
/// Applies pending changes on disk (reloads clean buffers, reports
/// changed directories and conflicts), writes due recovery-journal
/// entries (see [`crate::journal`]) and returns every queued event.
/// entries (see [`crate::journal`]), moves finished search results into
/// events (see [`cmd::search`]) and returns every queued event.
/// Hosts call it regularly, e.g. every 100 ms from a background thread.
///
/// # Errors
@ -141,6 +142,7 @@ impl Engine {
guard::run(|| {
crate::workspace::pump(state);
crate::journal::tick(state, std::time::Instant::now());
cmd::search::pump(state);
Ok(())
})?;
Ok(state.take_events())
@ -373,6 +375,8 @@ mod tests {
"recovery.discard",
"recovery.list",
"recovery.restore",
"search.cancel",
"search.find",
"settings.get",
"settings.list",
"settings.reset",

View file

@ -144,6 +144,49 @@ pub enum Event {
/// Project-relative paths with `/`, sorted.
paths: Vec<String>,
},
/// Matches of a running search (`search.find`) in one file. A file is
/// reported once per search; files arrive in no particular order.
SearchResults {
/// The search, as `search.find` returned it.
search_id: u64,
/// Project-relative path of the file.
path: String,
/// The matches in line order; never empty.
matches: Vec<SearchMatch>,
},
/// A search ended; no `searchResults` for it follow.
SearchDone {
/// The search.
search_id: u64,
/// Files with reported matches.
files: usize,
/// Matches reported.
matches: usize,
/// The match cap stopped the search; more matches exist.
truncated: bool,
/// `search.cancel` (or closing the project) stopped the search.
cancelled: bool,
/// Wall time of the search in milliseconds.
elapsed_ms: u64,
},
}
/// One match of [`Event::SearchResults`]. Positions are 0-based; columns
/// and lengths count UTF-16 code units.
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct SearchMatch {
/// Line of the match.
pub line: usize,
/// Start of the match in the line.
pub column: usize,
/// Length of the match; never 0.
pub length: usize,
/// The line without its line break, or a window of at most 300
/// characters around the match for longer lines.
pub preview: String,
/// Start of the match inside `preview`.
pub preview_column: usize,
}
/// Severity of an [`Event::Notification`].
@ -224,6 +267,39 @@ mod tests {
);
}
#[test]
fn search_shapes() {
let e = Event::SearchResults {
search_id: 2,
path: "src/a.php".into(),
matches: vec![super::SearchMatch {
line: 0,
column: 6,
length: 4,
preview: "<?php echo".into(),
preview_column: 6,
}],
};
assert_eq!(
serde_json::to_value(e).unwrap(),
json!({"kind": "searchResults", "searchId": 2, "path": "src/a.php", "matches": [
{"line": 0, "column": 6, "length": 4, "preview": "<?php echo", "previewColumn": 6}
]})
);
let e = Event::SearchDone {
search_id: 2,
files: 1,
matches: 1,
truncated: false,
cancelled: false,
elapsed_ms: 12,
};
assert_eq!(
serde_json::to_value(e).unwrap(),
json!({"kind": "searchDone", "searchId": 2, "files": 1, "matches": 1, "truncated": false, "cancelled": false, "elapsedMs": 12})
);
}
#[test]
fn ui_command_shape() {
let e = Event::UiCommand {

View file

@ -23,6 +23,6 @@ pub use buffers::{BufferId, LineEndingName, OpenBuffer};
pub use command::{Command, CommandInfo, CommandSpec, Effect, Outcome};
pub use engine::{Engine, EngineConfig};
pub use error::{EngineError, ErrorPayload};
pub use event::{Event, NotificationLevel};
pub use event::{Event, NotificationLevel, SearchMatch};
pub use registry::Registry;
pub use state::State;

View file

@ -35,6 +35,8 @@ pub struct State {
pub engine_commands: BTreeSet<&'static str>,
/// UI commands the frontend registered (`ui.registerCommands`), by id.
pub ui_commands: BTreeMap<String, UiCommand>,
/// Running searches (`search.find`).
pub searches: crate::cmd::search::Searches,
/// Number of notifications emitted, for their ids.
notifications: u64,
/// Events queued for the host, oldest first.

View file

@ -0,0 +1,207 @@
//! `search.find` and `search.cancel`: results arrive through the pump.
#![allow(
clippy::unwrap_used,
clippy::expect_used,
clippy::panic,
clippy::indexing_slicing
)]
use std::fs;
use std::time::{Duration, Instant};
use lutra_engine::cmd::search::MAX_RESULTS_PER_PUMP;
use lutra_engine::{Engine, Event};
use serde_json::{Value, json};
use tempfile::TempDir;
/// A temp folder with `files`, opened in a fresh engine.
fn open_project(files: &[(&str, &str)]) -> (TempDir, Engine) {
let dir = tempfile::tempdir().unwrap();
for (path, text) in files {
let abs = dir.path().join(path);
fs::create_dir_all(abs.parent().unwrap()).unwrap();
fs::write(abs, text).unwrap();
}
let mut engine = Engine::new().unwrap();
engine
.execute("project.open", json!({"path": dir.path()}))
.unwrap();
(dir, engine)
}
fn find(engine: &mut Engine, params: Value) -> u64 {
let out = engine.execute("search.find", params).unwrap();
assert!(out.events.is_empty(), "results only come from the pump");
out.value["searchId"].as_u64().unwrap()
}
/// Pumps until `searchDone` for `id`; returns the results (sorted by path)
/// and the done event, and the number of pumps.
fn collect(engine: &mut Engine, id: u64) -> (Vec<Event>, Event, usize) {
let deadline = Instant::now() + Duration::from_secs(10);
let mut results = Vec::new();
let mut pumps = 0;
while Instant::now() < deadline {
pumps += 1;
for event in engine.pump_vfs().unwrap() {
match &event {
Event::SearchResults { search_id, .. } if *search_id == id => results.push(event),
Event::SearchDone { search_id, .. } if *search_id == id => {
results.sort_by_key(path);
return (results, event, pumps);
}
_ => {}
}
}
std::thread::sleep(Duration::from_millis(10));
}
panic!("no searchDone within 10 s");
}
fn path(event: &Event) -> String {
match event {
Event::SearchResults { path, .. } => path.clone(),
_ => String::new(),
}
}
#[test]
fn finds_in_files_and_dirty_buffers() {
let (_dir, mut engine) = open_project(&[
("src/a.php", "<?php\necho 'needle';\n"),
("src/b.php", "<?php\n"),
("vendor/x/c.php", "<?php // needle\n"),
("node_modules/d.js", "needle\n"),
]);
let out = engine
.execute("file.open", json!({"path": "src/b.php"}))
.unwrap();
let buffer = out.value["id"].as_u64().unwrap();
engine
.execute(
"buffer.edit",
json!({"id": buffer, "baseVersion": 1, "edits": [{"from": 6, "to": 6, "insert": "$needle = 1;\n"}]}),
)
.unwrap();
let id = find(&mut engine, json!({"query": "NEEDLE"}));
let (results, done, _) = collect(&mut engine, id);
let paths: Vec<String> = results.iter().map(path).collect();
assert_eq!(paths, ["src/a.php", "src/b.php", "vendor/x/c.php"]);
let Event::SearchResults { matches, .. } = &results[1] else {
panic!()
};
assert_eq!(
(matches[0].line, matches[0].column, matches[0].length),
(1, 1, 6)
);
assert_eq!(matches[0].preview, "$needle = 1;");
let Event::SearchDone {
files,
matches,
truncated,
cancelled,
..
} = done
else {
panic!()
};
assert_eq!((files, matches, truncated, cancelled), (3, 3, false, false));
let id2 = find(
&mut engine,
json!({"query": "needle", "caseSensitive": true, "include": ["src/*.php"]}),
);
assert!(id2 > id);
let (results, _, _) = collect(&mut engine, id2);
assert_eq!(results.len(), 2);
}
#[test]
fn invalid_queries_fail_with_invalid_params() {
let (_dir, mut engine) = open_project(&[("a.php", "x")]);
for params in [
json!({"query": "(", "regex": true}),
json!({"query": ""}),
json!({"query": "x", "include": ["["]}),
json!({"query": "x", "bogus": 1}),
json!({}),
] {
let err = engine.execute("search.find", params.clone()).unwrap_err();
assert_eq!(err.code(), "invalid_params", "{params}");
}
assert_eq!(engine.state().searches.running(), 0);
engine.execute("project.close", json!({})).unwrap();
let err = engine
.execute("search.find", json!({"query": "x"}))
.unwrap_err();
assert_eq!(err.code(), "disabled");
}
#[test]
fn cancel_and_project_close_end_searches() {
let dir = tempfile::tempdir().unwrap();
lutra_testkit::generate_search_tree(dir.path(), 2_000);
let mut engine = Engine::new().unwrap();
engine
.execute("project.open", json!({"path": dir.path()}))
.unwrap();
let id = find(&mut engine, json!({"query": "function"}));
let out = engine
.execute("search.cancel", json!({"searchId": id}))
.unwrap();
assert_eq!(out.value["cancelled"], true);
let (_, done, _) = collect(&mut engine, id);
let Event::SearchDone {
cancelled, matches, ..
} = done
else {
panic!()
};
assert!(
cancelled || matches == 10_000,
"cancelled unless it had already finished"
);
let out = engine
.execute("search.cancel", json!({"searchId": id}))
.unwrap();
assert_eq!(out.value["cancelled"], false);
let id = find(&mut engine, json!({"query": "function"}));
engine.execute("project.close", json!({})).unwrap();
let (results, done, _) = collect(&mut engine, id);
assert!(results.is_empty());
assert!(matches!(
done,
Event::SearchDone {
cancelled: true,
..
}
));
assert_eq!(engine.state().searches.running(), 0);
}
#[test]
fn pumps_spread_many_results() {
let dir = tempfile::tempdir().unwrap();
for i in 0..(MAX_RESULTS_PER_PUMP + 100) {
fs::write(dir.path().join(format!("f{i}.txt")), "hit\n").unwrap();
}
let mut engine = Engine::new().unwrap();
engine
.execute("project.open", json!({"path": dir.path()}))
.unwrap();
let id = find(&mut engine, json!({"query": "hit"}));
std::thread::sleep(Duration::from_millis(500));
let first = engine.pump_vfs().unwrap();
let batch = first
.iter()
.filter(|e| matches!(e, Event::SearchResults { .. }))
.count();
assert!(batch <= MAX_RESULTS_PER_PUMP, "{batch}");
let (rest, done, _) = collect(&mut engine, id);
assert_eq!(batch + rest.len(), MAX_RESULTS_PER_PUMP + 100);
assert!(matches!(done, Event::SearchDone { files, .. } if files == MAX_RESULTS_PER_PUMP + 100));
}