From 053441d066b96aa3b27f0bb5ffbeea23886ef203 Mon Sep 17 00:00:00 2001 From: danny Date: Sat, 17 Oct 2026 12:53:33 +0200 Subject: [PATCH] M2.5: symbol index and go-to commands crates/index: walks the project with the search/VFS rules (vendor as library), parses .php files from tree-sitter outlines on a worker pool off the engine lock, updates incrementally from disk and buffers through an updater thread, caps at 200,000 files, and answers fuzzy file, class and symbol queries (nucleo-matcher, MPL-2.0 as a separate crate). Engine: index.status, goto.file, goto.class, goto.symbol; events indexProgress and indexReady drained by the pump; the index starts on project.open and is cancelled on close. Tests: unit, incremental, property (random queries and sources), engine and TCP control tests, release timing on a generated 20k-file project (3.9-4.2 s). Co-Authored-By: Claude Opus 5.5 --- Cargo.lock | 19 + crates/control/tests/goto.rs | 170 ++++++++ crates/engine/Cargo.toml | 1 + crates/engine/src/cmd/goto.rs | 511 +++++++++++++++++++++++ crates/engine/src/cmd/mod.rs | 5 + crates/engine/src/cmd/project.rs | 1 + crates/engine/src/engine.rs | 8 +- crates/engine/src/event.rs | 39 ++ crates/engine/src/state.rs | 2 + crates/engine/src/workspace.rs | 4 +- crates/engine/tests/goto.rs | 336 ++++++++++++++++ crates/index/Cargo.toml | 9 + crates/index/src/build.rs | 257 ++++++++++++ crates/index/src/lib.rs | 145 ++++++- crates/index/src/query.rs | 667 +++++++++++++++++++++++++++++++ crates/index/src/symbol.rs | 313 +++++++++++++++ crates/index/src/updater.rs | 192 +++++++++ crates/index/src/walk.rs | 193 +++++++++ crates/index/tests/index.rs | 222 ++++++++++ crates/index/tests/perf.rs | 78 ++++ crates/index/tests/properties.rs | 88 ++++ crates/testkit/src/lib.rs | 2 + crates/testkit/src/php.rs | 176 ++++++++ docs/architecture.md | 16 +- docs/gaps.md | 5 +- docs/target-app-parity.md | 5 +- 26 files changed, 3455 insertions(+), 9 deletions(-) create mode 100644 crates/control/tests/goto.rs create mode 100644 crates/engine/src/cmd/goto.rs create mode 100644 crates/engine/tests/goto.rs create mode 100644 crates/index/src/build.rs create mode 100644 crates/index/src/query.rs create mode 100644 crates/index/src/symbol.rs create mode 100644 crates/index/src/updater.rs create mode 100644 crates/index/src/walk.rs create mode 100644 crates/index/tests/index.rs create mode 100644 crates/index/tests/perf.rs create mode 100644 crates/index/tests/properties.rs create mode 100644 crates/testkit/src/php.rs diff --git a/Cargo.lock b/Cargo.lock index 0a895b6..efb22cb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2582,6 +2582,7 @@ dependencies = [ name = "lutra-engine" version = "0.0.0" dependencies = [ + "lutra-index", "lutra-search", "lutra-testkit", "lutra-text", @@ -2596,6 +2597,15 @@ dependencies = [ [[package]] name = "lutra-index" version = "0.0.0" +dependencies = [ + "ignore", + "lutra-syntax", + "lutra-testkit", + "lutra-text", + "nucleo-matcher", + "proptest", + "tempfile", +] [[package]] name = "lutra-lsp" @@ -2853,6 +2863,15 @@ dependencies = [ "bitflags 2.13.2", ] +[[package]] +name = "nucleo-matcher" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf33f538733d1a5a3494b836ba913207f14d9d4a1d3cd67030c5061bdd2cac85" +dependencies = [ + "memchr", +] + [[package]] name = "num-conv" version = "0.2.2" diff --git a/crates/control/tests/goto.rs b/crates/control/tests/goto.rs new file mode 100644 index 0000000..08a789a --- /dev/null +++ b/crates/control/tests/goto.rs @@ -0,0 +1,170 @@ +//! The symbol index over a real TCP control connection: `indexProgress` +//! and `indexReady` arrive as event lines from the pump, `index.status` and +//! the go-to commands answer. + +#![allow( + clippy::unwrap_used, + clippy::expect_used, + clippy::indexing_slicing, + clippy::panic +)] + +use std::io::{BufRead, BufReader, Write}; +use std::net::TcpStream; +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::time::Duration; + +use lutra_testkit::EngineBackend; +use serde_json::{Value, json}; + +/// A client that keeps responses and event lines apart. +struct Client { + reader: BufReader, + writer: TcpStream, + events: Vec, +} + +impl Client { + fn connect(addr: std::net::SocketAddr) -> Self { + let stream = TcpStream::connect(addr).unwrap(); + stream + .set_read_timeout(Some(Duration::from_secs(30))) + .unwrap(); + Self { + writer: stream.try_clone().unwrap(), + reader: BufReader::new(stream), + events: Vec::new(), + } + } + + fn line(&mut self) -> Value { + let mut line = String::new(); + self.reader.read_line(&mut line).expect("line within 30 s"); + serde_json::from_str(&line).unwrap() + } + + fn call(&mut self, request: Value) -> Value { + self.writer + .write_all(format!("{request}\n").as_bytes()) + .unwrap(); + loop { + let line = self.line(); + if line.get("event").is_some() { + self.events.push(line["event"].clone()); + } else { + return line; + } + } + } + + /// Reads event lines until `indexReady`; returns the `indexProgress` + /// events and the ready event. + fn until_ready(&mut self) -> (Vec, Value) { + let mut progress = Vec::new(); + let mut pending = std::mem::take(&mut self.events).into_iter(); + loop { + let event = match pending.next() { + Some(e) => e, + None => { + let line = self.line(); + let Some(event) = line.get("event") else { + continue; + }; + event.clone() + } + }; + match event["kind"].as_str() { + Some("indexProgress") => progress.push(event), + Some("indexReady") => return (progress, event), + _ => {} + } + } + } +} + +fn execute(command: &str, params: Value) -> Value { + json!({"id": command, "method": "engine.execute", "params": {"command": command, "params": params}}) +} + +/// Pumps the backend every 20 ms, like the desktop's pump thread, until +/// dropped. +struct Pump(Arc); + +impl Pump { + fn start(backend: Arc) -> Self { + let stop = Arc::new(AtomicBool::new(false)); + let flag = stop.clone(); + std::thread::spawn(move || { + while !flag.load(Ordering::Relaxed) { + backend.pump(); + std::thread::sleep(Duration::from_millis(20)); + } + }); + Self(stop) + } +} + +impl Drop for Pump { + fn drop(&mut self) { + self.0.store(true, Ordering::Relaxed); + } +} + +#[test] +fn index_builds_and_go_to_answers_over_tcp() { + let dir = tempfile::tempdir().unwrap(); + let project = lutra_testkit::generate_php_project(dir.path(), 2_000); + let backend = Arc::new(EngineBackend::new()); + let server = lutra_control::start(0, backend.clone()).unwrap(); + let mut client = Client::connect(server.local_addr()); + client.call(json!({"id": 1, "method": "events.subscribe"})); + let r = client.call(execute("project.open", json!({"path": dir.path()}))); + assert!(r["result"]["root"].is_string(), "{r}"); + let _pump = Pump::start(backend.clone()); + + let (progress, ready) = client.until_ready(); + for event in &progress { + assert_eq!(event["total"], project.indexed_files, "{event}"); + assert!(event["done"].as_u64().unwrap() <= project.indexed_files as u64); + } + assert_eq!(ready["files"], project.indexed_files); + assert!(ready["symbols"].as_u64().unwrap() > project.classes as u64); + assert!(ready["elapsedMs"].is_u64()); + + let r = client.call(execute("index.status", Value::Null)); + assert_eq!(r["result"]["state"], "idle", "{r}"); + assert_eq!(r["result"]["files"], project.indexed_files); + + let r = client.call(execute( + "goto.class", + json!({"query": "Service10", "limit": 5}), + )); + let items = r["result"]["items"].as_array().unwrap(); + assert_eq!(r["result"]["indexing"], false); + assert_eq!(items[0]["fqn"], "App\\Module000\\Service10", "{r}"); + assert_eq!(items[0]["path"], "src/Module000/Service10.php"); + assert_eq!(items[0]["kind"], "class"); + assert_eq!(items[0]["library"], false); + + let r = client.call(execute("goto.class", json!({"query": "Widget1"}))); + let first = &r["result"]["items"][0]; + assert_eq!(first["library"], true, "{r}"); + assert!(first["path"].as_str().unwrap().starts_with("vendor/")); + + let r = client.call(execute( + "goto.symbol", + json!({"query": "Service10::handle3"}), + )); + let items = r["result"]["items"].as_array().unwrap(); + assert_eq!(items[0]["fqn"], "App\\Module000\\Service10::handle3", "{r}"); + assert_eq!(items[0]["container"], "App\\Module000\\Service10"); + + let r = client.call(execute("goto.file", json!({"query": "service10.php"}))); + let first = &r["result"]["items"][0]; + assert_eq!(first["path"], "src/Module000/Service10.php", "{r}"); + assert!(first["matches"].is_array()); + + let r = client.call(execute("goto.file", json!({"query": 7}))); + assert_eq!(r["error"]["code"], "invalid_params", "{r}"); +} diff --git a/crates/engine/Cargo.toml b/crates/engine/Cargo.toml index ec1f13c..7dcefac 100644 --- a/crates/engine/Cargo.toml +++ b/crates/engine/Cargo.toml @@ -12,6 +12,7 @@ publish.workspace = true layer = 4 [dependencies] +lutra-index.workspace = true lutra-search.workspace = true lutra-text.workspace = true lutra-vfs.workspace = true diff --git a/crates/engine/src/cmd/goto.rs b/crates/engine/src/cmd/goto.rs new file mode 100644 index 0000000..af780b8 --- /dev/null +++ b/crates/engine/src/cmd/goto.rs @@ -0,0 +1,511 @@ +//! The symbol index and the go-to commands: `index.status`, `goto.file`, +//! `goto.class` and `goto.symbol`. +//! +//! # Index lifecycle +//! [`sync`] runs on every [`crate::Engine::pump_vfs`] and before every +//! index command. When a project is open and no index exists for its root, +//! it starts one: [`lutra_index::spawn_build`] walks and parses the project +//! on background threads, and an [`Updater`] thread handles later changes. +//! Both send [`Update`]s over one channel; [`sync`] merges them into the +//! in-memory [`Index`] in [`State`], so the engine lock is only held for +//! cheap inserts, never while files are walked or parsed. Progress becomes +//! [`Event::IndexProgress`] (at most every [`PROGRESS_EVERY`]) and the end +//! of the build [`Event::IndexReady`]. Closing the project (or opening +//! another one) drops the index and cancels its threads. +//! +//! # Incremental updates +//! - Disk: [`note_disk_changes`] turns the workspace's VFS events into +//! updater jobs (created or modified files are re-read, created or +//! renamed directories rescanned) and drops removed paths at once. The +//! project watcher is not recursive (see [`crate::workspace`]), so changes +//! in directories that no client listed and no buffer pins are not seen +//! until the next project open. +//! - Buffers: when an open buffer's version changes, its text is +//! re-indexed instead of the file on disk; when it is closed with unsaved +//! changes, the file on disk is re-read. Only files the index already +//! lists (or, while building, may still list) are re-indexed from +//! buffers, so a buffer of an ignored file never enters the index. +//! - Updates from the updater win over build results for the same path +//! that arrive later. +//! +//! # Queries +//! Queries run on the current index, never wait for the build, and report +//! `indexing: true` while it runs. See [`lutra_index::Index`] for matching +//! and ranking. + +use std::collections::{BTreeMap, HashSet}; +use std::path::PathBuf; +use std::sync::mpsc::{Receiver, TryRecvError, channel}; +use std::time::{Duration, Instant}; + +use lutra_index::{CancelToken, Index, Job, SymbolHit, Update, Updater}; +use lutra_vfs::{EntryKind, VfsEvent}; +use serde::Deserialize; +use serde_json::{Value, json}; + +use crate::buffers::BufferId; +use crate::cmd::params::parse; +use crate::workspace::path_string; +use crate::{Command, CommandSpec, EngineError, Event, Outcome, State}; + +/// Shortest time between two [`Event::IndexProgress`]. +pub const PROGRESS_EVERY: Duration = Duration::from_millis(250); + +/// Results of a go-to query when the params give no `limit`. +pub const DEFAULT_LIMIT: usize = 50; + +/// Most updates merged by one [`sync`]; the rest wait for the next one, so +/// a pump never holds the engine lock for long. +pub const MAX_UPDATES_PER_SYNC: usize = 4_096; + +/// The index of the open project, if one is being built or ready. +#[derive(Debug, Default)] +pub struct IndexState { + /// The index and its background threads. + running: Option, +} + +impl IndexState { + /// Whether the initial build is still running. + pub fn indexing(&self) -> bool { + self.running.as_ref().is_some_and(|r| r.building) + } + + /// The index, if a project is open. + pub fn index(&self) -> Option<&Index> { + self.running.as_ref().map(|r| &r.index) + } +} + +/// The index of one project root and its background work. +#[derive(Debug)] +struct Running { + /// The project root. + root: PathBuf, + /// The files and symbols known so far. + index: Index, + /// Stops the build and the updater. + cancel: CancelToken, + /// Updates from the build and the updater. + updates: Receiver, + /// Re-indexes changed files; `None` if its thread could not start. + updater: Option, + /// The initial build has not ended yet. + building: bool, + /// When the build started. + started: Instant, + /// The latest progress not yet emitted. + progress: Option<(usize, usize)>, + /// When progress was last emitted. + progress_at: Option, + /// Paths the updater reported while the build runs; later build results + /// for them are stale. + touched: HashSet, + /// The path and version each open buffer was last seen at. + buffers: BTreeMap, + /// Paths whose index entry comes from an unsaved buffer, so disk + /// changes don't replace it. + overlay: HashSet, +} + +impl Drop for Running { + /// Cancels the build and the updater; their threads end at the next + /// file. + fn drop(&mut self) { + self.cancel.cancel(); + } +} + +impl Running { + /// Starts indexing `root`. Threads that cannot start leave an index that + /// stays empty (build) or is not updated (updater); queries still work. + fn start(root: PathBuf) -> Self { + let (tx, updates) = channel(); + let cancel = CancelToken::new(); + let building = lutra_index::spawn_build( + root.clone(), + lutra_index::default_threads(), + cancel.clone(), + tx.clone(), + ) + .is_ok(); + let updater = Updater::spawn(root.clone(), cancel.clone(), tx).ok(); + Self { + root, + index: Index::new(), + cancel, + updates, + updater, + building, + started: Instant::now(), + progress: None, + progress_at: None, + touched: HashSet::new(), + buffers: BTreeMap::new(), + overlay: HashSet::new(), + } + } + + /// Queues `job` for the updater. + fn send(&self, job: Job) { + if let Some(updater) = &self.updater { + updater.send(job); + } + } + + /// Merges `update` into the index; returns the event it causes. + fn apply(&mut self, update: Update) -> Option { + match update { + Update::Files { files, live } => { + for file in files { + if live { + if self.building { + self.touched.insert(file.path.clone()); + } + } else if self.touched.contains(&file.path) { + continue; + } + self.index.insert(file); + } + None + } + Update::Removed { path } => { + self.index.remove(&path); + if self.building { + self.touched.insert(path); + } + None + } + Update::Progress { done, total } => { + self.progress = Some((done, total)); + None + } + Update::Ready(summary) => Some(self.ready(summary.elapsed)), + } + } + + /// Ends the build after `elapsed` and returns [`Event::IndexReady`]. + fn ready(&mut self, elapsed: Duration) -> Event { + self.building = false; + self.progress = None; + self.touched.clear(); + Event::IndexReady { + files: self.index.len(), + symbols: self.index.symbol_count(), + elapsed_ms: u64::try_from(elapsed.as_millis()).unwrap_or(u64::MAX), + } + } +} + +/// Brings the index in step with the state: starts it for a newly open +/// project, drops it without one, merges finished updates, re-indexes +/// changed buffers and emits progress and readiness events. Called by the +/// pump and by every index command. +pub fn sync(state: &mut State) { + let root = state + .project + .as_ref() + .map(|w| w.project.root().to_path_buf()); + let Some(root) = root else { + state.index.running = None; + return; + }; + if state.index.running.as_ref().is_none_or(|r| r.root != root) { + state.index.running = Some(Running::start(root)); + } + let Some(running) = state.index.running.as_mut() else { + return; + }; + let mut events = Vec::new(); + for _ in 0..MAX_UPDATES_PER_SYNC { + match running.updates.try_recv() { + Ok(update) => events.extend(running.apply(update)), + Err(TryRecvError::Empty) => break, + Err(TryRecvError::Disconnected) => { + if running.building { + let elapsed = running.started.elapsed(); + events.push(running.ready(elapsed)); + } + break; + } + } + } + if let Some((done, total)) = running.progress + && running.building + && running + .progress_at + .is_none_or(|at| at.elapsed() >= PROGRESS_EVERY) + { + running.progress = None; + running.progress_at = Some(Instant::now()); + events.push(Event::IndexProgress { done, total }); + } + sync_buffers(state); + for event in events { + state.emit(event); + } +} + +/// Re-indexes buffers whose text changed since the last sync, and re-reads +/// the files of closed buffers that were indexed from unsaved text. +fn sync_buffers(state: &mut State) { + let Some(running) = state.index.running.as_mut() else { + return; + }; + let mut seen = BTreeMap::new(); + for id in state.buffers.ids() { + let Some(open) = state.buffers.get(id) else { + continue; + }; + let path = path_string(&open.path); + let dirty = open.dirty(); + let changed = match running.buffers.get(&id) { + Some((old_path, version)) => *version != open.version || *old_path != path, + None => dirty, + }; + let known = running.index.contains(&path) || running.building; + if changed && known { + if dirty { + running.overlay.insert(path.clone()); + } + running.send(Job::File { + path: path.clone(), + text: Some(open.buffer.clone()), + }); + } + if !dirty { + running.overlay.remove(&path); + } + seen.insert(id, (path, open.version)); + } + let closed: Vec = running + .buffers + .iter() + .filter(|(id, _)| !seen.contains_key(id)) + .map(|(_, (path, _))| path.clone()) + .collect(); + for path in closed { + if running.overlay.remove(&path) { + running.send(Job::File { path, text: None }); + } + } + running.buffers = seen; +} + +/// Turns changes on disk into index updates: removed paths are dropped at +/// once, created and modified files are re-read, created directories and +/// rename targets are rescanned. Files indexed from an unsaved buffer keep +/// the buffer's text. Does nothing without an index. +pub fn note_disk_changes(state: &mut State, changes: &[VfsEvent]) { + let Some(running) = state.index.running.as_mut() else { + return; + }; + for change in changes { + match change { + VfsEvent::Created { path, kind } => { + let path = path_string(path); + if *kind == EntryKind::Directory { + running.send(Job::Dir { path }); + } else if !running.overlay.contains(&path) { + running.send(Job::File { path, text: None }); + } + } + VfsEvent::Modified { path } => { + let path = path_string(path); + if !running.overlay.contains(&path) { + running.send(Job::File { path, text: None }); + } + } + VfsEvent::Removed { path } => { + let path = path_string(path); + running.apply(Update::Removed { path }); + } + VfsEvent::Renamed { from, to } => { + running.apply(Update::Removed { + path: path_string(from), + }); + running.send(Job::Dir { + path: path_string(to), + }); + } + VfsEvent::Failed(_) => {} + } + } +} + +/// Drops the index and cancels its threads; `project.close` calls it. +pub fn close(state: &mut State) { + state.index.running = None; +} + +/// `index.status`: whether the index is building and how big it is. +pub struct Status; + +impl Command for Status { + fn spec(&self) -> &CommandSpec { + &CommandSpec { + id: "index.status", + label: "Index Status", + menu_path: None, + default_shortcut: None, + params: "none; returns { state: \"idle\" | \"indexing\", files, symbols }; events indexProgress { done, total } (at most every 250 ms) and indexReady { files, symbols, elapsedMs } report the build after project.open", + } + } + + fn run(&self, state: &mut State, _params: Value) -> Result { + sync(state); + let index = state.index.index(); + Ok(Outcome::value(json!({ + "state": if state.index.indexing() { "indexing" } else { "idle" }, + "files": index.map_or(0, Index::len), + "symbols": index.map_or(0, Index::symbol_count), + }))) + } +} + +/// Params of the go-to commands. +#[derive(Deserialize)] +#[serde(deny_unknown_fields, rename_all = "camelCase")] +struct QueryParams { + /// What to look for. + query: String, + /// Most results. + limit: Option, +} + +/// Parses the params of the go-to command `id`, brings the index up to +/// date and returns the query and the limit. +fn prepare(state: &mut State, id: &str, params: Value) -> Result<(String, usize), EngineError> { + let p: QueryParams = parse(id, params)?; + sync(state); + Ok((p.query, p.limit.unwrap_or(DEFAULT_LIMIT))) +} + +/// The spec of a go-to command. +const fn goto_spec(id: &'static str, label: &'static str, params: &'static str) -> CommandSpec { + CommandSpec { + id, + label, + menu_path: None, + default_shortcut: None, + params, + } +} + +/// `goto.file`: fuzzy search over the project's file paths. +pub struct File; + +impl Command for File { + fn spec(&self) -> &CommandSpec { + const SPEC: CommandSpec = goto_spec( + "goto.file", + "Go to File", + "{ query, limit? (default 50, at most 1000) }; returns { items: [{ path, score, matches: [[start, end]] }], indexing }: fuzzy over project-relative paths, file names weighted higher, project files before vendor files; matches are UTF-16 ranges in path; indexing is true while the index is still being built (partial results)", + ); + &SPEC + } + + fn enabled(&self, state: &State) -> bool { + state.has_project() + } + + fn run(&self, state: &mut State, params: Value) -> Result { + let (query, limit) = prepare(state, self.spec().id, params)?; + let items: Vec = state + .index + .index() + .map(|index| index.find_files(&query, limit)) + .unwrap_or_default() + .into_iter() + .map(|hit| json!({ "path": hit.path, "score": hit.score, "matches": hit.matches })) + .collect(); + Ok(Outcome::value( + json!({ "items": items, "indexing": state.index.indexing() }), + )) + } +} + +/// `goto.class`: fuzzy search over classes, interfaces, traits and enums. +pub struct Class; + +impl Command for Class { + fn spec(&self) -> &CommandSpec { + const SPEC: CommandSpec = goto_spec( + "goto.class", + "Go to Class", + "{ query, limit? (default 50, at most 1000) }; returns { items: [{ name, fqn, kind, path, line, column, library }], indexing }: classes, interfaces, traits and enums by name (by FQN when the query has a backslash), project before library; line and column (UTF-16) point at the name", + ); + &SPEC + } + + fn enabled(&self, state: &State) -> bool { + state.has_project() + } + + fn run(&self, state: &mut State, params: Value) -> Result { + let (query, limit) = prepare(state, self.spec().id, params)?; + let items: Vec = state + .index + .index() + .map(|index| index.find_classes(&query, limit)) + .unwrap_or_default() + .iter() + .map(|hit| symbol_item(hit, false)) + .collect(); + Ok(Outcome::value( + json!({ "items": items, "indexing": state.index.indexing() }), + )) + } +} + +/// `goto.symbol`: fuzzy search over functions, methods, properties, +/// constants and enum cases. +pub struct Symbol; + +impl Command for Symbol { + fn spec(&self) -> &CommandSpec { + const SPEC: CommandSpec = goto_spec( + "goto.symbol", + "Go to Symbol", + "{ query, limit? (default 50, at most 1000) }; returns { items: [{ name, fqn, kind, container, path, line, column, library }], indexing }: functions, methods, properties, constants and enum cases by name; Class::member matches the class and the member; container is the declaring class FQN or null", + ); + &SPEC + } + + fn enabled(&self, state: &State) -> bool { + state.has_project() + } + + fn run(&self, state: &mut State, params: Value) -> Result { + let (query, limit) = prepare(state, self.spec().id, params)?; + let items: Vec = state + .index + .index() + .map(|index| index.find_symbols(&query, limit)) + .unwrap_or_default() + .iter() + .map(|hit| symbol_item(hit, true)) + .collect(); + Ok(Outcome::value( + json!({ "items": items, "indexing": state.index.indexing() }), + )) + } +} + +/// The JSON item of a class or symbol hit, with `container` for symbols. +fn symbol_item(hit: &SymbolHit<'_>, container: bool) -> Value { + let s = hit.symbol; + let mut item = json!({ + "name": s.name, + "fqn": s.fqn, + "kind": s.kind.as_str(), + "path": hit.path, + "line": s.line, + "column": s.column, + "library": hit.library, + }); + if container && let Some(map) = item.as_object_mut() { + map.insert("container".into(), json!(s.container)); + } + item +} diff --git a/crates/engine/src/cmd/mod.rs b/crates/engine/src/cmd/mod.rs index 0c24c0c..073f7cf 100644 --- a/crates/engine/src/cmd/mod.rs +++ b/crates/engine/src/cmd/mod.rs @@ -3,6 +3,7 @@ mod app; mod buffer; mod file; +pub mod goto; mod params; mod project; mod recovery; @@ -42,5 +43,9 @@ pub fn register_builtins(registry: &mut Registry) -> Result<(), EngineError> { registry.register(Box::new(ui::RegisterCommands))?; registry.register(Box::new(search::Find))?; registry.register(Box::new(search::Cancel))?; + registry.register(Box::new(goto::Status))?; + registry.register(Box::new(goto::File))?; + registry.register(Box::new(goto::Class))?; + registry.register(Box::new(goto::Symbol))?; Ok(()) } diff --git a/crates/engine/src/cmd/project.rs b/crates/engine/src/cmd/project.rs index cd34271..a94b379 100644 --- a/crates/engine/src/cmd/project.rs +++ b/crates/engine/src/cmd/project.rs @@ -58,6 +58,7 @@ pub fn close_project(state: &mut State, force: bool) -> Result<(), EngineError> } state.journal = None; state.project = None; + crate::cmd::goto::close(state); state.emit(Event::ProjectClosed); Ok(()) } diff --git a/crates/engine/src/engine.rs b/crates/engine/src/engine.rs index 66ecba7..10f77bd 100644 --- a/crates/engine/src/engine.rs +++ b/crates/engine/src/engine.rs @@ -131,7 +131,8 @@ impl Engine { /// Applies pending changes on disk (reloads clean buffers, reports /// changed directories and conflicts), writes due recovery-journal /// entries (see [`crate::journal`]), moves finished search results into - /// events (see [`cmd::search`]) and returns every queued event. + /// events (see [`cmd::search`]), merges index updates (see + /// [`cmd::goto`]) and returns every queued event. /// Hosts call it regularly, e.g. every 100 ms from a background thread. /// /// # Errors @@ -143,6 +144,7 @@ impl Engine { crate::workspace::pump(state); crate::journal::tick(state, std::time::Instant::now()); cmd::search::pump(state); + cmd::goto::sync(state); Ok(()) })?; Ok(state.take_events()) @@ -368,6 +370,10 @@ mod tests { "file.open", "file.save", "file.saveAll", + "goto.class", + "goto.file", + "goto.symbol", + "index.status", "project.children", "project.close", "project.open", diff --git a/crates/engine/src/event.rs b/crates/engine/src/event.rs index fc8e6c6..ab41522 100644 --- a/crates/engine/src/event.rs +++ b/crates/engine/src/event.rs @@ -169,6 +169,24 @@ pub enum Event { /// Wall time of the search in milliseconds. elapsed_ms: u64, }, + /// The symbol index is being built after `project.open`; sent at most + /// every 250 ms while it runs. + IndexProgress { + /// Files indexed so far. + done: usize, + /// Files found in the project. + total: usize, + }, + /// The initial build of the symbol index ended (or was cut short); go-to + /// queries now see the whole project. + IndexReady { + /// Files in the index. + files: usize, + /// Symbols in the index. + symbols: usize, + /// Wall time of the build in milliseconds. + elapsed_ms: u64, + }, } /// One match of [`Event::SearchResults`]. Positions are 0-based; columns @@ -312,6 +330,27 @@ mod tests { ); } + #[test] + fn index_shapes() { + let e = Event::IndexProgress { + done: 10, + total: 20, + }; + assert_eq!( + serde_json::to_value(e).unwrap(), + json!({"kind": "indexProgress", "done": 10, "total": 20}) + ); + let e = Event::IndexReady { + files: 20, + symbols: 300, + elapsed_ms: 1500, + }; + assert_eq!( + serde_json::to_value(e).unwrap(), + json!({"kind": "indexReady", "files": 20, "symbols": 300, "elapsedMs": 1500}) + ); + } + #[test] fn recovery_resolved_shape() { let e = Event::RecoveryResolved { diff --git a/crates/engine/src/state.rs b/crates/engine/src/state.rs index 47a8557..626f5d0 100644 --- a/crates/engine/src/state.rs +++ b/crates/engine/src/state.rs @@ -37,6 +37,8 @@ pub struct State { pub ui_commands: BTreeMap, /// Running searches (`search.find`). pub searches: crate::cmd::search::Searches, + /// The symbol index of the open project (`index.status`, `goto.*`). + pub index: crate::cmd::goto::IndexState, /// Number of notifications emitted, for their ids. notifications: u64, /// Events queued for the host, oldest first. diff --git a/crates/engine/src/workspace.rs b/crates/engine/src/workspace.rs index 9597102..2f8204c 100644 --- a/crates/engine/src/workspace.rs +++ b/crates/engine/src/workspace.rs @@ -234,7 +234,8 @@ pub fn sync_buffer_dirs(state: &mut State) { workspace.pin_dirs(dirs); } -/// Applies pending changes on disk to the state: queues +/// Applies pending changes on disk to the state: hands them to the symbol +/// index ([`crate::cmd::goto::note_disk_changes`]), queues /// [`Event::TreeChanged`] for affected directories, reloads clean buffers /// whose file changed, and queues [`Event::FileChangedOnDisk`] for dirty /// buffers and for buffers whose file disappeared. @@ -244,6 +245,7 @@ pub fn pump(state: &mut State) { }; let (changes, failure) = workspace.drain(); let root = workspace.project.root().to_path_buf(); + crate::cmd::goto::note_disk_changes(state, &changes); let mut dirs: Vec = Vec::new(); let mut add_dir = |path: &Path| { let dir = path.parent().map(Path::to_path_buf).unwrap_or_default(); diff --git a/crates/engine/tests/goto.rs b/crates/engine/tests/goto.rs new file mode 100644 index 0000000..7355abc --- /dev/null +++ b/crates/engine/tests/goto.rs @@ -0,0 +1,336 @@ +//! The symbol index in the engine: `index.status`, the go-to commands, +//! progress and readiness events, and incremental updates from disk and +//! buffers. + +#![allow( + clippy::unwrap_used, + clippy::expect_used, + clippy::panic, + clippy::indexing_slicing +)] + +use std::fs; +use std::path::Path; +use std::time::{Duration, Instant}; + +use lutra_engine::{Engine, Event}; +use proptest::prelude::*; +use serde_json::{Value, json}; +use tempfile::TempDir; + +fn write(root: &Path, rel: &str, text: &str) { + let abs = root.join(rel); + fs::create_dir_all(abs.parent().unwrap()).unwrap(); + fs::write(abs, text).unwrap(); +} + +/// A temp folder with `files`, opened in a fresh engine. +fn open_project(files: &[(&str, &str)]) -> (TempDir, Engine) { + let dir = tempfile::tempdir().unwrap(); + for (path, text) in files { + write(dir.path(), path, text); + } + let mut engine = Engine::new().unwrap(); + engine + .execute("project.open", json!({"path": dir.path()})) + .unwrap(); + (dir, engine) +} + +fn run(engine: &mut Engine, command: &str, params: Value) -> Value { + engine.execute(command, params).unwrap().value +} + +/// Pumps until `indexReady`; returns every event seen. +fn until_ready(engine: &mut Engine) -> Vec { + let deadline = Instant::now() + Duration::from_secs(30); + let mut seen = Vec::new(); + while Instant::now() < deadline { + let events = engine.pump_vfs().unwrap(); + let ready = events.iter().any(|e| matches!(e, Event::IndexReady { .. })); + seen.extend(events); + if ready { + return seen; + } + std::thread::sleep(Duration::from_millis(10)); + } + panic!("no indexReady within 30 s"); +} + +/// Pumps until `query` with `command` gives `count` items. +fn until_count(engine: &mut Engine, command: &str, query: &str, count: usize) -> Value { + let deadline = Instant::now() + Duration::from_secs(10); + loop { + engine.pump_vfs().unwrap(); + let out = run(engine, command, json!({"query": query})); + if out["items"].as_array().unwrap().len() == count { + return out; + } + assert!( + Instant::now() < deadline, + "{command} {query}: wanted {count} items, got {out}" + ); + std::thread::sleep(Duration::from_millis(20)); + } +} + +fn names(out: &Value, field: &str) -> Vec { + out["items"] + .as_array() + .unwrap() + .iter() + .map(|i| i[field].as_str().unwrap().to_string()) + .collect() +} + +#[test] +fn commands_need_a_project_and_status_is_idle_without_one() { + let mut engine = Engine::new().unwrap(); + assert_eq!( + run(&mut engine, "index.status", Value::Null), + json!({"state": "idle", "files": 0, "symbols": 0}) + ); + for command in ["goto.file", "goto.class", "goto.symbol"] { + let err = engine.execute(command, json!({"query": "a"})).unwrap_err(); + assert_eq!(err.code(), "disabled"); + } +} + +#[test] +fn builds_in_the_background_and_answers_go_to_queries() { + let (_dir, mut engine) = open_project(&[ + (".gitignore", "/var/\n/vendor/\n"), + ( + "src/Models/User.php", + " Some((*files, *symbols)), + _ => None, + }) + .unwrap(); + assert_eq!(ready, (5, 7)); + assert_eq!( + run(&mut engine, "index.status", Value::Null), + json!({"state": "idle", "files": 5, "symbols": 7}) + ); + + let files = run(&mut engine, "goto.file", json!({"query": "user"})); + assert_eq!(files["indexing"], false); + assert_eq!( + names(&files, "path"), + vec![ + "src/Models/User.php", + "templates/user.html.twig", + "vendor/acme/lib/User.php", + ] + ); + assert_eq!(files["items"][0]["matches"], json!([[11, 15]])); + assert!(files["items"][0]["score"].as_u64().unwrap() > 0); + + let classes = run(&mut engine, "goto.class", json!({"query": "user"})); + assert_eq!( + classes["items"], + json!([ + {"name": "User", "fqn": "App\\Models\\User", "kind": "class", "path": "src/Models/User.php", "line": 3, "column": 12, "library": false}, + {"name": "User", "fqn": "Acme\\User", "kind": "interface", "path": "vendor/acme/lib/User.php", "line": 3, "column": 10, "library": true}, + ]) + ); + + let symbols = run( + &mut engine, + "goto.symbol", + json!({"query": "find", "limit": 1}), + ); + assert_eq!( + symbols["items"], + json!([{"name": "find", "fqn": "App\\Models\\User::find", "kind": "method", "container": "App\\Models\\User", "path": "src/Models/User.php", "line": 7, "column": 20, "library": false}]) + ); + let function = run(&mut engine, "goto.symbol", json!({"query": "format_user"})); + assert_eq!(function["items"][0]["container"], Value::Null); + let members = run(&mut engine, "goto.symbol", json!({"query": "User::"})); + assert_eq!( + names(&members, "name"), + vec!["ROLE", "find", "$email", "find"] + ); + let empty = run(&mut engine, "goto.class", json!({"query": ""})); + assert_eq!(empty["items"], json!([])); + + for bad in [ + json!({}), + json!({"query": 3}), + json!({"query": "a", "limit": -1}), + json!({"query": "a", "extra": true}), + ] { + let err = engine.execute("goto.file", bad).unwrap_err(); + assert_eq!(err.code(), "invalid_params"); + } +} + +#[test] +fn reports_progress_and_partial_results_while_indexing() { + let dir = tempfile::tempdir().unwrap(); + let project = lutra_testkit::generate_php_project(dir.path(), 3_000); + let mut engine = Engine::new().unwrap(); + engine + .execute("project.open", json!({"path": dir.path()})) + .unwrap(); + let status = run(&mut engine, "index.status", Value::Null); + assert_eq!(status["state"], "indexing"); + let early = run(&mut engine, "goto.class", json!({"query": "Service"})); + assert_eq!(early["indexing"], true); + let events = until_ready(&mut engine); + let progress: Vec<(usize, usize)> = events + .iter() + .filter_map(|e| match e { + Event::IndexProgress { done, total } => Some((*done, *total)), + _ => None, + }) + .collect(); + assert!( + progress + .iter() + .all(|(_, total)| *total == project.indexed_files) + ); + assert!(progress.windows(2).all(|w| w[0].0 <= w[1].0)); + let status = run(&mut engine, "index.status", Value::Null); + assert_eq!(status["state"], "idle"); + assert_eq!(status["files"], project.indexed_files); + let classes = run( + &mut engine, + "goto.class", + json!({"query": "Service1", "limit": 1000}), + ); + assert_eq!(classes["indexing"], false); + assert!(!classes["items"].as_array().unwrap().is_empty()); +} + +#[test] +fn follows_buffers_and_disk_changes() { + let (dir, mut engine) = open_project(&[("src/A.php", " Summary { + let started = Instant::now(); + let walked = walk(root, None, threads, MAX_FILES, cancel); + let walk_time = started.elapsed(); + let total = walked.files.len(); + let (mut php, rest): (Vec, Vec) = walked + .files + .into_iter() + .partition(|f| is_php(&f.path) && f.size <= MAX_PARSE_BYTES); + let mut done = 0usize; + let mut batch = Vec::with_capacity(BATCH_FILES); + for found in rest { + batch.push(IndexedFile { + library: is_library(&found.path), + path: found.path, + symbols: Vec::new(), + }); + if batch.len() >= BATCH_FILES { + done = done.saturating_add(batch.len()); + send(Update::Files { + files: std::mem::take(&mut batch), + live: false, + }); + } + } + done = done.saturating_add(batch.len()); + if !batch.is_empty() { + send(Update::Files { + files: batch, + live: false, + }); + } + send(Update::Progress { done, total }); + php.sort_by(|a, b| { + is_library(&a.path) + .cmp(&is_library(&b.path)) + .then(b.size.cmp(&a.size)) + }); + let shared = Shared { + files: &php, + next: AtomicUsize::new(0), + done: AtomicUsize::new(done), + total, + symbols: AtomicUsize::new(0), + parsed: AtomicUsize::new(0), + parsed_bytes: AtomicU64::new(0), + cancel, + send, + }; + let workers = threads.clamp(1, php.len().max(1)); + std::thread::scope(|scope| { + for _ in 0..workers { + scope.spawn(|| shared.work()); + } + }); + Summary { + files: shared.done.load(Ordering::Relaxed), + symbols: shared.symbols.load(Ordering::Relaxed), + parsed: shared.parsed.load(Ordering::Relaxed), + parsed_bytes: shared.parsed_bytes.load(Ordering::Relaxed), + capped: walked.capped, + cancelled: cancel.is_cancelled(), + walk: walk_time, + elapsed: started.elapsed(), + } +} + +/// Runs [`build`] on a thread of its own and sends its updates, then +/// [`Update::Ready`], over `updates`. Updates sent after the receiver is +/// gone are dropped; the caller cancels the build when it stops listening. +/// +/// # Errors +/// The thread could not be started. +pub fn spawn_build( + root: PathBuf, + threads: usize, + cancel: CancelToken, + updates: Sender, +) -> std::io::Result<()> { + std::thread::Builder::new() + .name("lutra-index".into()) + .spawn(move || { + let send = |update| { + let _ = updates.send(update); + }; + let summary = build(&root, threads, &cancel, &send); + send(Update::Ready(summary)); + }) + .map(|_| ()) +} + +/// State the workers of one build share. +struct Shared<'a> { + /// PHP files to parse, in order. + files: &'a [Found], + /// Index of the next file to take. + next: AtomicUsize, + /// Files indexed and sent. + done: AtomicUsize, + /// Files found by the walk. + total: usize, + /// Symbols found. + symbols: AtomicUsize, + /// Files parsed. + parsed: AtomicUsize, + /// Bytes parsed. + parsed_bytes: AtomicU64, + /// Stops the workers. + cancel: &'a CancelToken, + /// Receives the batches. + send: &'a (dyn Fn(Update) + Sync), +} + +impl Shared<'_> { + /// One worker: takes files until none are left or the build is + /// cancelled, and sends them in batches. + fn work(&self) { + let mut parser = Parser::new().ok(); + let mut batch = Vec::with_capacity(WORKER_BATCH); + let mut last = Instant::now(); + while !self.cancel.is_cancelled() { + let i = self.next.fetch_add(1, Ordering::Relaxed); + let Some(found) = self.files.get(i) else { + break; + }; + let (file, bytes) = index_file(&mut parser, found); + if let Some(bytes) = bytes { + self.parsed.fetch_add(1, Ordering::Relaxed); + self.parsed_bytes.fetch_add(bytes, Ordering::Relaxed); + } + self.symbols + .fetch_add(file.symbols.len(), Ordering::Relaxed); + batch.push(file); + if batch.len() >= WORKER_BATCH || last.elapsed() >= FLUSH_AFTER { + self.flush(&mut batch); + last = Instant::now(); + } + } + self.flush(&mut batch); + } + + /// Sends `batch` and the progress after it. + fn flush(&self, batch: &mut Vec) { + if batch.is_empty() { + return; + } + let count = batch.len(); + (self.send)(Update::Files { + files: std::mem::take(batch), + live: false, + }); + let done = self + .done + .fetch_add(count, Ordering::Relaxed) + .saturating_add(count); + (self.send)(Update::Progress { + done, + total: self.total, + }); + } +} + +/// Reads and indexes one file. Returns the file and the bytes parsed, +/// `None` when it was listed without symbols: it is not a PHP file, could +/// not be read, grew past [`MAX_PARSE_BYTES`], no parser is available, or +/// parsing panicked. +pub(crate) fn index_file(parser: &mut Option, found: &Found) -> (IndexedFile, Option) { + let empty = || IndexedFile { + path: found.path.clone(), + library: is_library(&found.path), + symbols: Vec::new(), + }; + if !is_php(&found.path) { + return (empty(), None); + } + let Some(text) = read_source(&found.abs) else { + return (empty(), None); + }; + match guarded_index(parser, found.path.clone(), &text) { + Some(file) => (file, Some(text.len() as u64)), + None => (empty(), None), + } +} + +/// [`index_text`] with a guard: `None` when no parser is available or +/// parsing panicked (a bug in the parser; the parser is replaced so the +/// next file gets a fresh one, and the index goes on without this file's +/// symbols). +pub(crate) fn guarded_index( + parser: &mut Option, + path: String, + text: &str, +) -> Option { + let p = parser.as_mut()?; + match catch_unwind(AssertUnwindSafe(|| index_text(p, path, text))) { + Ok(file) => Some(file), + Err(_) => { + *parser = Parser::new().ok(); + None + } + } +} + +/// The text of the source file at `path`: at most [`MAX_PARSE_BYTES`], +/// without a UTF-8 byte order mark, invalid UTF-8 replaced. `None` if it +/// cannot be read or is larger. +pub(crate) fn read_source(path: &Path) -> Option { + let file = std::fs::File::open(path).ok()?; + let mut bytes = Vec::new(); + file.take(MAX_PARSE_BYTES.saturating_add(1)) + .read_to_end(&mut bytes) + .ok()?; + if bytes.len() as u64 > MAX_PARSE_BYTES { + return None; + } + let body = bytes.strip_prefix(b"\xEF\xBB\xBF").unwrap_or(&bytes); + Some(String::from_utf8_lossy(body).into_owned()) +} diff --git a/crates/index/src/lib.rs b/crates/index/src/lib.rs index 32936aa..849974e 100644 --- a/crates/index/src/lib.rs +++ b/crates/index/src/lib.rs @@ -1,4 +1,145 @@ //! Symbol index for Lutra (layer L2). //! -//! Stores declarations and usages for go-to and find-usages, starting with -//! symbols extracted from tree-sitter trees. +//! Lists every file of a project and the declarations of its PHP files +//! (classes, interfaces, traits, enums, functions, methods, properties, +//! constants, enum cases), extracted from tree-sitter outlines +//! ([`lutra_syntax::outline`]), and answers fuzzy go-to queries over them. +//! +//! # Which files +//! The walk follows the rules of `lutra-vfs` and `lutra-search`: +//! `.gitignore`, `.ignore`, `.git/info/exclude` and the global git excludes +//! apply without a git repository; `.git` and every `node_modules` directory +//! are skipped; the root's `vendor/` is walked even when an ignore file +//! lists it and its files are marked as library code; dot files are listed; +//! symbolic links are not followed. Every file is listed for +//! [`Index::find_files`]; only `.php` files are parsed, and only those up to +//! [`MAX_PARSE_BYTES`]. At most [`MAX_FILES`] files are indexed. +//! +//! # Building +//! [`build`] walks the project, then parses the PHP files on a pool of +//! worker threads (one parser each, one file in memory each), project files +//! before library files and larger files first within each group. It +//! reports [`Update`]s through a callback: batches of indexed files, +//! progress, and a final [`Summary`]. [`spawn_build`] runs it on a thread of +//! its own and sends the updates over a channel. [`Updater`] re-indexes +//! single files and directories later, from disk or from an editor buffer. +//! A [`CancelToken`] stops both between files. +//! +//! # Robustness +//! Source files are untrusted: reads are capped, invalid UTF-8 is replaced, +//! broken PHP yields the declarations that could be recognised, symbols per +//! file are capped ([`MAX_SYMBOLS_PER_FILE`]), and a panic while parsing one +//! file (a bug) loses that file's symbols, not the index. + +mod build; +mod query; +mod symbol; +mod updater; +mod walk; + +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::time::Duration; + +pub use build::{build, spawn_build}; +pub use query::{FileHit, Index, MAX_LIMIT, MAX_QUERY_CHARS, SymbolHit}; +pub use symbol::{IndexedFile, Symbol, SymbolKind, index_text, is_library, symbols_from_outline}; +pub use updater::{Job, Updater}; + +/// Most files one index holds. +pub const MAX_FILES: usize = 200_000; + +/// Largest PHP file parsed, in bytes (2 MiB). Larger files are listed for +/// [`Index::find_files`] without symbols: tree-sitter-php parses about +/// 2–3 MB/s, so one such file would hold a worker for a second or more, and +/// files this size are generated code rather than declarations to navigate. +pub const MAX_PARSE_BYTES: u64 = 2 * 1024 * 1024; + +/// Most symbols kept from one file; the rest are dropped. +pub const MAX_SYMBOLS_PER_FILE: usize = 20_000; + +/// Most worker threads of one build. +pub const MAX_THREADS: usize = 32; + +/// Files per [`Update::Files`] batch of a build. +pub const BATCH_FILES: usize = 256; + +/// What a build or an [`Updater`] reports. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Update { + /// Indexed files that replace any earlier entry with the same path. + Files { + /// The files. + files: Vec, + /// From an [`Updater`] (a change after the build started), so newer + /// than anything the build reports for the same path. + live: bool, + }, + /// A file or directory is gone: drop `path` and everything below it. + Removed { + /// Project-relative path with `/` separators. + path: String, + }, + /// Build progress: `done` of `total` files are indexed. + Progress { + /// Files indexed so far. + done: usize, + /// Files found by the walk. + total: usize, + }, + /// The build ended; nothing from it follows. + Ready(Summary), +} + +/// How a build ended. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub struct Summary { + /// Files indexed. + pub files: usize, + /// Symbols found in them. + pub symbols: usize, + /// PHP files parsed. + pub parsed: usize, + /// Bytes of PHP parsed. + pub parsed_bytes: u64, + /// The walk found more than [`MAX_FILES`] files; the rest are left out. + pub capped: bool, + /// The build was cancelled before it finished. + pub cancelled: bool, + /// Wall time of the walk alone. + pub walk: Duration, + /// Wall time from start to end. + pub elapsed: Duration, +} + +/// Stops a running build or updater. Clones share the flag. +#[derive(Debug, Clone, Default)] +pub struct CancelToken(Arc); + +impl CancelToken { + /// A token that is not cancelled. + pub fn new() -> Self { + Self::default() + } + + /// Asks the work to stop at the next file. + pub fn cancel(&self) { + self.0.store(true, Ordering::Relaxed); + } + + /// Whether [`CancelToken::cancel`] was called. + pub fn is_cancelled(&self) -> bool { + self.0.load(Ordering::Relaxed) + } +} + +/// Worker threads for a build on this machine: one less than the cores, +/// so the UI and the engine keep a core, at least one and at most +/// [`MAX_THREADS`]. +pub fn default_threads() -> usize { + std::thread::available_parallelism() + .map(|n| n.get()) + .unwrap_or(2) + .saturating_sub(1) + .clamp(1, MAX_THREADS) +} diff --git a/crates/index/src/query.rs b/crates/index/src/query.rs new file mode 100644 index 0000000..e244ac1 --- /dev/null +++ b/crates/index/src/query.rs @@ -0,0 +1,667 @@ +//! The index in memory and the fuzzy go-to queries over it. +//! +//! Matching uses `nucleo-matcher` (fzf-style scoring: consecutive +//! characters, word starts and path separators score higher). The query is +//! split at whitespace and every word must match; no other syntax is +//! special. Lower-case query characters match either case, upper-case ones +//! only themselves (smart case). Project files and symbols always rank +//! before library ones; within each group the higher score wins, then the +//! shorter name, then the name in byte order, so results are stable. + +use std::cmp::Ordering; +use std::collections::BTreeMap; + +use nucleo_matcher::pattern::{AtomKind, CaseMatching, Normalization, Pattern}; +use nucleo_matcher::{Config, Matcher, Utf32Str}; + +use crate::{IndexedFile, MAX_FILES, Symbol}; + +/// Most results one query returns. +pub const MAX_LIMIT: usize = 1_000; + +/// Longest query, in characters; the rest is ignored. +pub const MAX_QUERY_CHARS: usize = 256; + +/// Score factor of a match inside the file name over a match in the whole +/// path, so `user` finds `User.php` before `user/Repository.php`. +const NAME_WEIGHT: u32 = 2; + +/// Files plus symbols from which a query is scanned on several threads. +const PARALLEL_FROM: usize = 20_000; + +/// The files and symbols of a project. +#[derive(Debug, Default)] +pub struct Index { + /// Files by project-relative path. + files: BTreeMap, + /// Symbols in all files. + symbols: usize, +} + +/// A file found by [`Index::find_files`]. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct FileHit { + /// Project-relative path with `/` separators. + pub path: String, + /// Match score; higher is better. + pub score: u32, + /// Matched character ranges in `path`, `[start, end)` in UTF-16 code + /// units, ascending and not overlapping. + pub matches: Vec<[usize; 2]>, + /// Library code (below `vendor/`). + pub library: bool, +} + +/// A symbol found by [`Index::find_classes`] or [`Index::find_symbols`]. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct SymbolHit<'a> { + /// The symbol. + pub symbol: &'a Symbol, + /// Project-relative path of its file. + pub path: &'a str, + /// Library code (below `vendor/`). + pub library: bool, + /// Match score; higher is better. + pub score: u32, +} + +impl Index { + /// An empty index. + pub fn new() -> Self { + Self::default() + } + + /// Number of files. + pub fn len(&self) -> usize { + self.files.len() + } + + /// Whether the index holds no files. + pub fn is_empty(&self) -> bool { + self.files.is_empty() + } + + /// Number of symbols in all files. + pub fn symbol_count(&self) -> usize { + self.symbols + } + + /// The file at the project-relative `path`. + pub fn get(&self, path: &str) -> Option<&IndexedFile> { + self.files.get(path) + } + + /// Whether the index holds the file at `path`. + pub fn contains(&self, path: &str) -> bool { + self.files.contains_key(path) + } + + /// Adds `file`, replacing the entry with the same path. A new file is + /// refused (returns `false`) when the index already holds + /// [`MAX_FILES`] files. + pub fn insert(&mut self, file: IndexedFile) -> bool { + if self.files.len() >= MAX_FILES && !self.files.contains_key(&file.path) { + return false; + } + self.symbols = self.symbols.saturating_add(file.symbols.len()); + if let Some(old) = self.files.insert(file.path.clone(), file) { + self.symbols = self.symbols.saturating_sub(old.symbols.len()); + } + true + } + + /// Drops the file at `path`, or every file below the directory `path`. + /// Returns the number of files dropped. + pub fn remove(&mut self, path: &str) -> usize { + let path = path.trim_matches('/'); + if let Some(old) = self.files.remove(path) { + self.symbols = self.symbols.saturating_sub(old.symbols.len()); + return 1; + } + let keys: Vec = if path.is_empty() { + self.files.keys().cloned().collect() + } else { + let start = format!("{path}/"); + let end = format!("{path}0"); + self.files + .range(start..end) + .map(|(k, _)| k.clone()) + .collect() + }; + for key in &keys { + if let Some(old) = self.files.remove(key) { + self.symbols = self.symbols.saturating_sub(old.symbols.len()); + } + } + keys.len() + } + + /// Drops everything. + pub fn clear(&mut self) { + self.files.clear(); + self.symbols = 0; + } + + /// Files whose project-relative path matches `query`, best first, at + /// most `limit` (capped at [`MAX_LIMIT`]). + /// + /// A query without `/` (or `\`, read as `/`) is matched against the + /// file name first, which scores [`NAME_WEIGHT`] times higher, and + /// against the whole path otherwise; a query with `/` against the whole + /// path only. An empty query finds nothing. + pub fn find_files(&self, query: &str, limit: usize) -> Vec { + let query = clamp(query).replace('\\', "/"); + let by_path = query.contains('/'); + let Some(mut fuzzy) = Fuzzy::new(&query, Config::DEFAULT.match_paths()) else { + return Vec::new(); + }; + let order = |a: &(&IndexedFile, u32, bool), b: &(&IndexedFile, u32, bool)| { + rank((a.0.library, a.1, &a.0.path), (b.0.library, b.1, &b.0.path)) + }; + let hits = self.scan( + limit, + || Fuzzy::new(&query, Config::DEFAULT.match_paths()), + |fuzzy, file, hits| { + let name = file_name(&file.path); + if !by_path && let Some(score) = fuzzy.score(name) { + hits.push((file, score.saturating_mul(NAME_WEIGHT), true)); + } else if let Some(score) = fuzzy.score(&file.path) { + hits.push((file, score, false)); + } + }, + order, + ); + hits.into_iter() + .map(|(file, score, on_name)| { + let name = file_name(&file.path); + let (haystack, base) = if on_name { + let prefix = file.path.len().saturating_sub(name.len()); + let base = file.path.get(..prefix).map_or(0, utf16_len); + (name, base) + } else { + (file.path.as_str(), 0) + }; + FileHit { + path: file.path.clone(), + score, + matches: ranges(haystack, &fuzzy.indices(haystack), base), + library: file.library, + } + }) + .collect() + } + + /// Classes, interfaces, traits and enums whose name matches `query`, + /// best first, at most `limit` (capped at [`MAX_LIMIT`]). A query with + /// `\` is matched against the fully qualified name instead; a leading + /// `\` is ignored. An empty query finds nothing. + pub fn find_classes(&self, query: &str, limit: usize) -> Vec> { + self.find_symbols_where(query, limit, true) + } + + /// Functions, methods, properties, constants and enum cases whose name + /// matches `query`, best first, at most `limit` (capped at + /// [`MAX_LIMIT`]). `Class::member` matches the container's short name + /// (or its FQN when the part before `::` has a `\`) and the member name + /// separately; `Class::` lists the members of matching classes. A query + /// with `\` and no `::` is matched against the fully qualified name. An + /// empty query finds nothing. + pub fn find_symbols(&self, query: &str, limit: usize) -> Vec> { + self.find_symbols_where(query, limit, false) + } + + /// The shared part of [`Index::find_classes`] (`classes`) and + /// [`Index::find_symbols`]. + fn find_symbols_where<'a>( + &'a self, + query: &str, + limit: usize, + classes: bool, + ) -> Vec> { + let query = clamp(query); + let query = query.trim().trim_start_matches('\\'); + let config = symbol_config(); + let split = if classes { + None + } else { + query.split_once("::") + }; + let (owner_query, member_query) = match split { + Some((owner, member)) => (Some(owner.trim_start_matches('\\')), member), + None => (None, query), + }; + let owner_by_fqn = owner_query.is_some_and(|q| q.contains('\\')); + let by_fqn = owner_query.is_none() && member_query.contains('\\'); + let matchers = || { + let owner = owner_query.and_then(|q| Fuzzy::new(q, config.clone())); + let member = Fuzzy::new(member_query, config.clone()); + (owner.is_some() || member.is_some()).then_some((owner, member)) + }; + if matchers().is_none() { + return Vec::new(); + } + let visit = |(owner, member): &mut (Option, Option), + file: &'a IndexedFile, + hits: &mut Vec>| { + for symbol in &file.symbols { + if symbol.kind.is_class_like() != classes { + continue; + } + let mut score = 0u32; + if let Some(owner) = owner.as_mut() { + let Some(container) = symbol.container.as_deref() else { + continue; + }; + let haystack = if owner_by_fqn { + container + } else { + container.rsplit('\\').next().unwrap_or(container) + }; + let Some(s) = owner.score(haystack) else { + continue; + }; + score = s; + } + if let Some(member) = member.as_mut() { + let haystack = if by_fqn { &symbol.fqn } else { &symbol.name }; + let Some(s) = member.score(haystack) else { + continue; + }; + score = score.saturating_add(s); + } + hits.push(SymbolHit { + symbol, + path: &file.path, + library: file.library, + score, + }); + } + }; + self.scan(limit, matchers, visit, |a, b| { + rank( + (a.library, a.score, &a.symbol.name), + (b.library, b.score, &b.symbol.name), + ) + .then_with(|| a.symbol.fqn.cmp(&b.symbol.fqn)) + .then_with(|| a.path.cmp(b.path)) + .then_with(|| a.symbol.line.cmp(&b.symbol.line)) + }) + } + + /// Runs `visit` over every file and returns the best `limit` hits in + /// `order`. Large indexes are scanned on [`crate::default_threads`] + /// threads, each with its own matcher state from `make` and its own + /// best hits, merged at the end; small ones on the calling thread. + /// Returns nothing when `make` gives no state (an empty query). + fn scan<'a, S, T: Send>( + &'a self, + limit: usize, + make: impl Fn() -> Option + Sync, + visit: impl Fn(&mut S, &'a IndexedFile, &mut Vec) + Sync, + order: impl Fn(&T, &T) -> Ordering + Sync, + ) -> Vec { + let files: Vec<&IndexedFile> = self.files.values().collect(); + let work = files.len().saturating_add(self.symbols); + let threads = if work < PARALLEL_FROM { + 1 + } else { + crate::default_threads() + }; + let chunk = files.len().div_ceil(threads).max(1); + let run = |part: &[&'a IndexedFile]| { + let mut hits = Vec::new(); + if let Some(mut state) = make() { + for file in part { + visit(&mut state, file, &mut hits); + } + } + best(&mut hits, limit, &order); + hits + }; + let mut hits: Vec = if threads <= 1 { + run(&files) + } else { + std::thread::scope(|scope| { + let handles: Vec<_> = files + .chunks(chunk) + .map(|part| scope.spawn(|| run(part))) + .collect(); + handles + .into_iter() + .flat_map(|h| h.join().unwrap_or_default()) + .collect() + }) + }; + best(&mut hits, limit, &order); + hits + } +} + +/// The ranking order: project before library, higher score first, then +/// shorter, then byte order. +fn rank(a: (bool, u32, &str), b: (bool, u32, &str)) -> Ordering { + a.0.cmp(&b.0) + .then_with(|| b.1.cmp(&a.1)) + .then_with(|| a.2.len().cmp(&b.2.len())) + .then_with(|| a.2.cmp(b.2)) +} + +/// Keeps the `limit` (at most [`MAX_LIMIT`]) first items of `hits` in +/// `order`, sorted. +fn best(hits: &mut Vec, limit: usize, order: impl Fn(&T, &T) -> Ordering) { + let limit = limit.min(MAX_LIMIT); + if limit == 0 { + hits.clear(); + return; + } + if hits.len() > limit { + hits.select_nth_unstable_by(limit.saturating_sub(1), &order); + hits.truncate(limit); + } + hits.sort_by(order); +} + +/// The matcher configuration for symbol names: the default word +/// boundaries (camel case, `\`, `_`, `:` and other punctuation start +/// words), and matches near the start score a little higher. +fn symbol_config() -> Config { + let mut config = Config::DEFAULT; + config.prefer_prefix = true; + config +} + +/// The first [`MAX_QUERY_CHARS`] characters of `query`. +fn clamp(query: &str) -> String { + query.chars().take(MAX_QUERY_CHARS).collect() +} + +/// The file name of a project-relative path. +fn file_name(path: &str) -> &str { + path.rsplit('/').next().unwrap_or(path) +} + +/// Length of `text` in UTF-16 code units. +fn utf16_len(text: &str) -> usize { + text.chars().map(char::len_utf16).sum() +} + +/// Turns sorted character indices into `text` into merged `[start, end)` +/// UTF-16 ranges, shifted by `base`. +fn ranges(text: &str, indices: &[u32], base: usize) -> Vec<[usize; 2]> { + let mut out: Vec<[usize; 2]> = Vec::new(); + let mut wanted = indices.iter().copied().peekable(); + let mut at = base; + for (i, ch) in text.chars().enumerate() { + let width = ch.len_utf16(); + if wanted + .peek() + .is_some_and(|&w| usize::try_from(w).is_ok_and(|w| w == i)) + { + wanted.next(); + match out.last_mut() { + Some(range) if range[1] == at => range[1] = at.saturating_add(width), + _ => out.push([at, at.saturating_add(width)]), + } + } + at = at.saturating_add(width); + } + out +} + +/// A compiled query with the matcher and buffers it needs. +struct Fuzzy { + /// The query words. + pattern: Pattern, + /// The matcher and its scratch memory. + matcher: Matcher, + /// Characters of the current haystack, for non-ASCII text. + chars: Vec, + /// Match indices, reused. + indices: Vec, +} + +impl Fuzzy { + /// The query `query` with `config`; `None` for a query without words. + fn new(query: &str, config: Config) -> Option { + let pattern = Pattern::new( + query, + CaseMatching::Smart, + Normalization::Smart, + AtomKind::Fuzzy, + ); + if pattern.atoms.is_empty() { + return None; + } + Some(Self { + pattern, + matcher: Matcher::new(config), + chars: Vec::new(), + indices: Vec::new(), + }) + } + + /// The score of `haystack`, `None` if it does not match. + fn score(&mut self, haystack: &str) -> Option { + let Self { + pattern, + matcher, + chars, + .. + } = self; + pattern.score(utf32(haystack, chars), matcher) + } + + /// The matched character indices in `haystack`, sorted, without + /// duplicates; empty if it does not match. + fn indices(&mut self, haystack: &str) -> Vec { + let Self { + pattern, + matcher, + chars, + indices, + } = self; + indices.clear(); + if pattern + .indices(utf32(haystack, chars), matcher, indices) + .is_none() + { + return Vec::new(); + } + indices.sort_unstable(); + indices.dedup(); + indices.clone() + } +} + +/// `text` as the matcher's string type, one entry per Unicode scalar +/// value, so match indices are character indices (`Utf32Str::new` would +/// merge grapheme clusters). +fn utf32<'a>(text: &'a str, chars: &'a mut Vec) -> Utf32Str<'a> { + if text.is_ascii() { + return Utf32Str::Ascii(text.as_bytes()); + } + chars.clear(); + chars.extend(text.chars()); + Utf32Str::Unicode(chars) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::SymbolKind; + + fn file(path: &str, symbols: &[(&str, &str, SymbolKind, Option<&str>)]) -> IndexedFile { + IndexedFile { + path: path.into(), + library: crate::is_library(path), + symbols: symbols + .iter() + .map(|(name, fqn, kind, container)| Symbol { + name: (*name).into(), + fqn: (*fqn).into(), + kind: *kind, + container: container.map(str::to_string), + line: 1, + column: 2, + }) + .collect(), + } + } + + fn sample() -> Index { + let mut index = Index::new(); + index.insert(file( + "src/Models/User.php", + &[ + ("User", "App\\Models\\User", SymbolKind::Class, None), + ( + "find", + "App\\Models\\User::find", + SymbolKind::Method, + Some("App\\Models\\User"), + ), + ( + "$name", + "App\\Models\\User::$name", + SymbolKind::Property, + Some("App\\Models\\User"), + ), + ], + )); + index.insert(file( + "src/user/Repository.php", + &[( + "Repository", + "App\\User\\Repository", + SymbolKind::Class, + None, + )], + )); + index.insert(file( + "vendor/acme/lib/src/User.php", + &[ + ("User", "Acme\\User", SymbolKind::Class, None), + ( + "find", + "Acme\\User::find", + SymbolKind::Method, + Some("Acme\\User"), + ), + ], + )); + index.insert(file("templates/user.html.twig", &[])); + index.insert(file( + "src/helpers.php", + &[("format_user", "format_user", SymbolKind::Function, None)], + )); + index + } + + #[test] + fn counts_and_removal() { + let mut index = sample(); + assert_eq!(index.len(), 5); + assert_eq!(index.symbol_count(), 7); + index.insert(file("src/helpers.php", &[])); + assert_eq!(index.symbol_count(), 6); + assert_eq!(index.remove("src/user"), 1); + assert_eq!(index.remove("src/Models/User.php"), 1); + assert_eq!(index.remove("src/Models"), 0); + assert_eq!(index.symbol_count(), 2); + assert!(index.contains("src/helpers.php")); + assert_eq!(index.remove(""), 3); + assert!(index.is_empty()); + assert_eq!(index.symbol_count(), 0); + } + + #[test] + fn prefix_removal_respects_segments() { + let mut index = Index::new(); + index.insert(file("src/a.php", &[])); + index.insert(file("src2/b.php", &[])); + index.insert(file("src-x/c.php", &[])); + assert_eq!(index.remove("src"), 1); + assert_eq!(index.len(), 2); + } + + #[test] + fn files_rank_names_then_paths_and_project_first() { + let index = sample(); + let paths: Vec = index + .find_files("user", 10) + .into_iter() + .map(|h| h.path) + .collect(); + assert_eq!( + paths, + vec![ + "src/Models/User.php", + "templates/user.html.twig", + "src/user/Repository.php", + "vendor/acme/lib/src/User.php", + ] + ); + let hit = &index.find_files("user", 1)[0]; + assert_eq!(hit.matches, vec![[11, 15]]); + let by_path = index.find_files("models/u", 10); + assert_eq!(by_path[0].path, "src/Models/User.php"); + assert!(index.find_files(" ", 10).is_empty()); + assert!(index.find_files("user", 0).is_empty()); + assert!(index.find_files("zzz", 10).is_empty()); + } + + #[test] + fn classes_and_symbols() { + let index = sample(); + let classes: Vec<(&str, bool)> = index + .find_classes("user", 10) + .iter() + .map(|h| (h.symbol.fqn.as_str(), h.library)) + .collect(); + assert_eq!( + classes, + vec![("App\\Models\\User", false), ("Acme\\User", true)] + ); + let fqn = index.find_classes("\\acme\\us", 10); + assert_eq!(fqn.len(), 1); + assert_eq!(fqn[0].symbol.fqn, "Acme\\User"); + let finds: Vec<&str> = index + .find_symbols("find", 10) + .iter() + .map(|h| h.symbol.fqn.as_str()) + .collect(); + assert_eq!(finds, vec!["App\\Models\\User::find", "Acme\\User::find"]); + let members: Vec<&str> = index + .find_symbols("User::", 10) + .iter() + .map(|h| h.symbol.name.as_str()) + .collect(); + assert_eq!(members, vec!["find", "$name", "find"]); + let scoped = index.find_symbols("acme\\user::f", 10); + assert_eq!(scoped.len(), 1); + assert!( + index + .find_symbols("user", 10) + .iter() + .all(|h| !h.symbol.kind.is_class_like()) + ); + assert!(index.find_symbols("::", 10).is_empty()); + } + + #[test] + fn match_ranges_count_utf16() { + assert_eq!(ranges("aéb", &[0, 1, 2], 0), vec![[0, 3]]); + assert_eq!(ranges("a😀b", &[2], 4), vec![[7, 8]]); + assert_eq!(ranges("abc", &[0, 2], 0), vec![[0, 1], [2, 3]]); + } + + #[test] + fn caps_the_number_of_files() { + let mut index = Index::new(); + for i in 0..MAX_FILES { + index.files.insert(format!("f{i}"), IndexedFile::default()); + } + assert!(!index.insert(file("new.php", &[]))); + assert!(index.insert(file("f1", &[]))); + } +} diff --git a/crates/index/src/symbol.rs b/crates/index/src/symbol.rs new file mode 100644 index 0000000..bdfd67b --- /dev/null +++ b/crates/index/src/symbol.rs @@ -0,0 +1,313 @@ +//! Symbols and indexed files, extracted from tree-sitter outlines. + +use lutra_syntax::{OutlineItem, OutlineKind, Parser}; + +use crate::{MAX_PARSE_BYTES, MAX_SYMBOLS_PER_FILE}; + +/// What a symbol declares. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum SymbolKind { + /// A class. + Class, + /// An interface. + Interface, + /// A trait. + Trait, + /// An enum. + Enum, + /// A function outside a class. + Function, + /// A method. + Method, + /// A property, declared or constructor-promoted. + Property, + /// A class constant or a `const` outside a class. + Constant, + /// A `case` of an enum. + EnumCase, +} + +impl SymbolKind { + /// The kind's name as the engine and the UI use it, the same as the + /// outline's: `class`, `interface`, `trait`, `enum`, `function`, + /// `method`, `property`, `constant` or `enumCase`. + pub fn as_str(self) -> &'static str { + match self { + Self::Class => "class", + Self::Interface => "interface", + Self::Trait => "trait", + Self::Enum => "enum", + Self::Function => "function", + Self::Method => "method", + Self::Property => "property", + Self::Constant => "constant", + Self::EnumCase => "enumCase", + } + } + + /// Whether the kind is a class, interface, trait or enum (Go to Class); + /// the others are found by Go to Symbol. + pub fn is_class_like(self) -> bool { + matches!( + self, + Self::Class | Self::Interface | Self::Trait | Self::Enum + ) + } + + /// The symbol kind of an outline kind; `None` for a namespace. + fn from_outline(kind: OutlineKind) -> Option { + Some(match kind { + OutlineKind::Namespace => return None, + OutlineKind::Class => Self::Class, + OutlineKind::Interface => Self::Interface, + OutlineKind::Trait => Self::Trait, + OutlineKind::Enum => Self::Enum, + OutlineKind::Function => Self::Function, + OutlineKind::Method => Self::Method, + OutlineKind::Property => Self::Property, + OutlineKind::Constant => Self::Constant, + OutlineKind::EnumCase => Self::EnumCase, + }) + } +} + +/// One declaration. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct Symbol { + /// The name as written: `Product`, `find`, `$id` (properties keep the + /// `$`), `MAX`. + pub name: String, + /// The fully qualified name without a leading `\`: `App\Models\Product` + /// for a class or function, `App\Models\Product::find` for a member. + pub fqn: String, + /// What it declares. + pub kind: SymbolKind, + /// The FQN of the class, interface, trait or enum that declares a + /// member; `None` for classes, functions and constants outside a class. + pub container: Option, + /// 0-based line of the name. + pub line: usize, + /// Column of the name in UTF-16 code units. + pub column: usize, +} + +/// A file in the index. +#[derive(Debug, Clone, PartialEq, Eq, Default)] +pub struct IndexedFile { + /// Project-relative path with `/` separators. + pub path: String, + /// Library code: the file is below the root's `vendor/`. + pub library: bool, + /// Declarations in source order; empty for files that are not parsed. + pub symbols: Vec, +} + +/// Whether the project-relative `path` is library code (the root's +/// `vendor/` or anything below it). +pub fn is_library(path: &str) -> bool { + path == "vendor" || path.starts_with("vendor/") +} + +/// Whether the project-relative `path` is a PHP file to parse. +pub(crate) fn is_php(path: &str) -> bool { + path.rsplit('/') + .next() + .and_then(|name| name.rsplit_once('.')) + .is_some_and(|(stem, ext)| !stem.is_empty() && ext.eq_ignore_ascii_case("php")) +} + +/// Indexes `text` as the file at `path`: parses it when it is a PHP file +/// of at most [`MAX_PARSE_BYTES`] and extracts its symbols; other files get +/// no symbols. +/// +/// # Parameters +/// - `parser`: reused between files. +/// - `path`: project-relative path with `/` separators. +/// - `text`: the file's text. +pub fn index_text(parser: &mut Parser, path: String, text: &str) -> IndexedFile { + let library = is_library(&path); + let parse = is_php(&path) && text.len() as u64 <= MAX_PARSE_BYTES; + let symbols = if parse { + let tree = parser.parse(text); + symbols_from_outline(&lutra_syntax::outline(&tree)) + } else { + Vec::new() + }; + IndexedFile { + path, + library, + symbols, + } +} + +/// The symbols declared in an outline, in source order, at most +/// [`MAX_SYMBOLS_PER_FILE`]. +/// +/// Namespaces give the FQN prefix of what they contain; classes, +/// interfaces, traits and enums are the container of their members. +/// Anonymous classes and their members are skipped: nothing can navigate to +/// them by name. +pub fn symbols_from_outline(items: &[OutlineItem]) -> Vec { + let mut out = Vec::new(); + collect(items, "", None, &mut out); + out +} + +/// Adds the symbols of `items` to `out`. +/// +/// # Parameters +/// - `namespace`: the enclosing namespace without leading or trailing `\`; +/// empty for the global namespace. +/// - `container`: the FQN of the enclosing class-like declaration. +fn collect(items: &[OutlineItem], namespace: &str, container: Option<&str>, out: &mut Vec) { + for item in items { + if out.len() >= MAX_SYMBOLS_PER_FILE { + return; + } + let Some(kind) = SymbolKind::from_outline(item.kind) else { + let name = item.name.trim_matches('\\'); + collect(&item.children, name, None, out); + continue; + }; + if item.is_anonymous() || item.name.is_empty() { + continue; + } + let (fqn, owner) = match container { + Some(owner) if !kind.is_class_like() && kind != SymbolKind::Function => { + (format!("{owner}::{}", item.name), Some(owner.to_string())) + } + _ => (qualify(namespace, &item.name), None), + }; + let start = item.selection_range.start; + out.push(Symbol { + name: item.name.clone(), + fqn: fqn.clone(), + kind, + container: owner, + line: start.line, + column: start.column, + }); + if kind.is_class_like() { + collect(&item.children, namespace, Some(&fqn), out); + } + } +} + +/// `name` qualified with `namespace`: `App\Product`, or `Product` in the +/// global namespace. +fn qualify(namespace: &str, name: &str) -> String { + let name = name.trim_start_matches('\\'); + if namespace.is_empty() { + name.to_string() + } else { + format!("{namespace}\\{name}") + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn symbols(source: &str) -> Vec { + let mut parser = Parser::new().unwrap(); + index_text(&mut parser, "src/a.php".into(), source).symbols + } + + #[test] + fn qualifies_classes_members_and_functions() { + let found = symbols( + ")> = found + .iter() + .map(|s| (s.fqn.clone(), s.kind.as_str(), s.container.clone())) + .collect(); + let product = Some("App\\Models\\Product".to_string()); + assert_eq!( + summary, + vec![ + ("App\\Models\\HasId".into(), "interface", None), + ("App\\Models\\Product".into(), "class", None), + ( + "App\\Models\\Product::MAX".into(), + "constant", + product.clone() + ), + ( + "App\\Models\\Product::$name".into(), + "property", + product.clone() + ), + ( + "App\\Models\\Product::__construct".into(), + "method", + product.clone() + ), + ( + "App\\Models\\Product::$id".into(), + "property", + product.clone() + ), + ("App\\Models\\Product::find".into(), "method", product), + ("App\\Models\\Status".into(), "enum", None), + ( + "App\\Models\\Status::Draft".into(), + "enumCase", + Some("App\\Models\\Status".into()) + ), + ("App\\Models\\helper".into(), "function", None), + ("App\\Models\\VERSION".into(), "constant", None), + ] + ); + let product = found.iter().find(|s| s.name == "Product").unwrap(); + assert_eq!((product.line, product.column), (4, 12)); + } + + #[test] + fn braced_and_global_namespaces() { + let found = symbols( + " = found.iter().map(|s| s.fqn.as_str()).collect(); + assert_eq!(fqns, vec!["A\\One", "Two", "three"]); + } + + #[test] + fn only_php_files_are_parsed() { + let mut parser = Parser::new().unwrap(); + let text = ", + }, + /// Re-index a directory: drop everything below it, then list it again + /// from disk. + Dir { + /// Project-relative path with `/` separators. + path: String, + }, +} + +impl Job { + /// The path the job is about. + fn path(&self) -> &str { + match self { + Self::File { path, .. } | Self::Dir { path } => path, + } + } +} + +/// A background thread that runs [`Job`]s one batch at a time and sends +/// the results as live [`Update`]s. Jobs queued while it works are +/// coalesced: only the last job per path runs, so a burst of keystrokes in +/// one buffer parses it once. The thread ends when the updater is dropped +/// or `cancel` fires. +#[derive(Debug)] +pub struct Updater { + /// Queues jobs for the thread. + jobs: Sender, +} + +impl Updater { + /// Starts the thread for the project at `root`. + /// + /// # Parameters + /// - `root`: the project root (absolute). + /// - `cancel`: stops the thread between jobs. + /// - `updates`: where results go, usually the build's channel. + /// + /// # Errors + /// The thread could not be started. + pub fn spawn( + root: PathBuf, + cancel: CancelToken, + updates: Sender, + ) -> std::io::Result { + let (jobs, queue) = channel(); + std::thread::Builder::new() + .name("lutra-index-update".into()) + .spawn(move || run(&root, &cancel, &queue, &updates))?; + Ok(Self { jobs }) + } + + /// Queues `job`. Returns `false` if the thread has ended. + pub fn send(&self, job: Job) -> bool { + self.jobs.send(job).is_ok() + } +} + +/// The updater thread's loop. +fn run( + root: &std::path::Path, + cancel: &CancelToken, + queue: &Receiver, + updates: &Sender, +) { + let mut parser = Parser::new().ok(); + while let Ok(first) = queue.recv() { + let mut jobs = vec![first]; + jobs.extend(queue.try_iter()); + for job in coalesce(jobs) { + if cancel.is_cancelled() { + return; + } + let sent = match job { + Job::File { + path, + text: Some(text), + } => { + let library = crate::is_library(&path); + let file = guarded_index(&mut parser, path.clone(), &text.text()).unwrap_or( + IndexedFile { + path, + library, + symbols: Vec::new(), + }, + ); + updates + .send(Update::Files { + files: vec![file], + live: true, + }) + .is_ok() + } + Job::File { path, text: None } => { + rescan(root, &path, false, cancel, &mut parser, updates) + } + Job::Dir { path } => rescan(root, &path, true, cancel, &mut parser, updates), + }; + if !sent { + return; + } + } + } +} + +/// Keeps the last job per path, in the order of those last jobs. +fn coalesce(jobs: Vec) -> Vec { + let mut last: HashMap = HashMap::new(); + for (i, job) in jobs.iter().enumerate() { + last.insert(job.path().to_string(), i); + } + jobs.into_iter() + .enumerate() + .filter(|(i, job)| last.get(job.path()) == Some(i)) + .map(|(_, job)| job) + .collect() +} + +/// Lists `path` from disk with the project's rules and indexes what is +/// found. A file that is no longer found is reported removed; a directory +/// is always reported removed first, so files gone from it disappear. +/// Returns `false` once the receiver is gone. +fn rescan( + root: &std::path::Path, + path: &str, + dir: bool, + cancel: &CancelToken, + parser: &mut Option, + updates: &Sender, +) -> bool { + let walked = walk(root, Some(path), 1, MAX_FILES, cancel); + if (dir || walked.files.is_empty()) + && updates + .send(Update::Removed { + path: path.to_string(), + }) + .is_err() + { + return false; + } + let mut batch = Vec::new(); + for found in &walked.files { + if cancel.is_cancelled() { + return true; + } + batch.push(index_file(parser, found).0); + if batch.len() >= RESCAN_BATCH + && updates + .send(Update::Files { + files: std::mem::take(&mut batch), + live: true, + }) + .is_err() + { + return false; + } + } + batch.is_empty() + || updates + .send(Update::Files { + files: batch, + live: true, + }) + .is_ok() +} diff --git a/crates/index/src/walk.rs b/crates/index/src/walk.rs new file mode 100644 index 0000000..9863633 --- /dev/null +++ b/crates/index/src/walk.rs @@ -0,0 +1,193 @@ +//! The walk over the project: which files the index lists (see the crate +//! docs for the rules). + +use std::path::{Path, PathBuf}; +use std::sync::Mutex; +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; + +use ignore::{DirEntry, WalkBuilder, WalkState}; + +use crate::CancelToken; +use crate::symbol::is_php; + +/// A file the walk found. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct Found { + /// Project-relative path with `/` separators. + pub path: String, + /// Absolute path. + pub abs: PathBuf, + /// Size in bytes for PHP files; 0 for other files, which are not read. + pub size: u64, +} + +/// What a walk returns. +#[derive(Debug, Default)] +pub(crate) struct Walked { + /// The files, in no particular order. + pub files: Vec, + /// More than the limit were found; the rest are left out. + pub capped: bool, +} + +/// Lists the files below `root`, or only those at or below the +/// project-relative `scope` (a file or a directory), with the project's +/// rules. +/// +/// # Parameters +/// - `root`: the project root (absolute). +/// - `scope`: `None` for the whole project. +/// - `threads`: walker threads. +/// - `limit`: most files to return. +/// - `cancel`: stops the walk. +/// +/// Entries that cannot be read are skipped, as the project tree skips +/// them. +pub(crate) fn walk( + root: &Path, + scope: Option<&str>, + threads: usize, + limit: usize, + cancel: &CancelToken, +) -> Walked { + let found = Mutex::new(Vec::new()); + let count = AtomicUsize::new(0); + let capped = AtomicBool::new(false); + let scope = scope.map(|s| s.trim_matches('/').to_string()); + walker(root, scope.clone(), threads) + .build_parallel() + .run(|| { + let found = &found; + let count = &count; + let capped = &capped; + let scope = scope.as_deref(); + Box::new(move |entry| { + if cancel.is_cancelled() || capped.load(Ordering::Relaxed) { + return WalkState::Quit; + } + let Ok(entry) = entry else { + return WalkState::Continue; + }; + if !entry.file_type().is_some_and(|t| t.is_file()) { + return WalkState::Continue; + } + let Some(rel) = relative(root, entry.path()) else { + return WalkState::Continue; + }; + if scope.is_some_and(|s| !inside(&rel, s)) { + return WalkState::Continue; + } + if count.fetch_add(1, Ordering::Relaxed) >= limit { + capped.store(true, Ordering::Relaxed); + return WalkState::Quit; + } + let size = if is_php(&rel) { + entry.metadata().map(|m| m.len()).unwrap_or(0) + } else { + 0 + }; + if let Ok(mut files) = found.lock() { + files.push(Found { + path: rel, + abs: entry.path().to_path_buf(), + size, + }); + } + WalkState::Continue + }) + }); + Walked { + files: found.into_inner().unwrap_or_default(), + capped: capped.load(Ordering::Relaxed), + } +} + +/// The walker over `root` with the project's rules, entering only +/// directories on the way to or inside `scope`. +fn walker(root: &Path, scope: Option, threads: usize) -> WalkBuilder { + let mut builder = WalkBuilder::new(root); + builder + .hidden(false) + .parents(false) + .ignore(true) + .git_ignore(true) + .git_exclude(true) + .git_global(true) + .require_git(false) + .follow_links(false) + .threads(threads.max(1)); + let vendor = root.join("vendor"); + let vendor_is_dir = std::fs::symlink_metadata(&vendor).is_ok_and(|m| m.is_dir()); + let vendor_in_scope = scope + .as_deref() + .is_none_or(|s| related("vendor", s) || s.is_empty()); + if vendor_is_dir && vendor_in_scope { + builder.add(&vendor); + } + let root = root.to_path_buf(); + builder.filter_entry(move |entry| keep_entry(&root, &vendor, scope.as_deref(), entry)); + builder +} + +/// Whether the walk enters (directory) or visits (file) `entry`: `.git` +/// and `node_modules` are skipped, the root's `vendor` is reached through +/// its own walk root, and with a scope only directories on the way to or +/// inside it are entered. +fn keep_entry(root: &Path, vendor: &Path, scope: Option<&str>, entry: &DirEntry) -> bool { + let name = entry.file_name(); + if name == ".git" { + return false; + } + let is_dir = entry.file_type().is_some_and(|t| t.is_dir()); + if is_dir && name == "node_modules" { + return false; + } + if entry.depth() == 1 && entry.path() == vendor { + return false; + } + if is_dir + && let Some(scope) = scope + && let Some(rel) = relative(root, entry.path()) + { + return related(&rel, scope); + } + true +} + +/// Whether `path` is `scope` or below it. +fn inside(path: &str, scope: &str) -> bool { + scope.is_empty() + || path == scope + || path + .strip_prefix(scope) + .is_some_and(|rest| rest.starts_with('/')) +} + +/// Whether the directory `dir` is on the way to `scope` or inside it. +fn related(dir: &str, scope: &str) -> bool { + inside(dir, scope) || inside(scope, dir) +} + +/// `path` relative to `root` with `/` separators; `None` outside the root +/// or for a name that is not valid Unicode. +fn relative(root: &Path, path: &Path) -> Option { + let rel = path.strip_prefix(root).ok()?; + let parts: Option> = rel.components().map(|c| c.as_os_str().to_str()).collect(); + Some(parts?.join("/")) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn scope_relations() { + assert!(inside("src/a.php", "src")); + assert!(inside("src", "src")); + assert!(!inside("src2/a.php", "src")); + assert!(inside("anything", "")); + assert!(related("src", "src/deep/a.php")); + assert!(related("src/deep/x", "src/deep")); + assert!(!related("lib", "src/a.php")); + } +} diff --git a/crates/index/tests/index.rs b/crates/index/tests/index.rs new file mode 100644 index 0000000..b752d7c --- /dev/null +++ b/crates/index/tests/index.rs @@ -0,0 +1,222 @@ +//! Building the index of a project on disk, incremental updates, and +//! cancellation. + +#![allow( + clippy::unwrap_used, + clippy::expect_used, + clippy::indexing_slicing, + clippy::panic +)] + +use std::fs; +use std::path::Path; +use std::sync::Mutex; +use std::sync::mpsc::{Receiver, channel}; +use std::time::Duration; + +use lutra_index::{CancelToken, Index, Job, Update, Updater, build, spawn_build}; + +/// Builds the index of `root` and applies every update. +fn build_index(root: &Path, threads: usize) -> (Index, lutra_index::Summary, Vec<(usize, usize)>) { + let updates = Mutex::new(Vec::new()); + let summary = build(root, threads, &CancelToken::new(), &|u| { + updates.lock().unwrap().push(u) + }); + let mut index = Index::new(); + let mut progress = Vec::new(); + for update in updates.into_inner().unwrap() { + match update { + Update::Files { files, live } => { + assert!(!live); + for file in files { + assert!(index.insert(file)); + } + } + Update::Progress { done, total } => progress.push((done, total)), + other => panic!("unexpected {other:?}"), + } + } + (index, summary, progress) +} + +/// Applies updates from `rx` until one batch arrives and `rx` is quiet. +fn apply_until_quiet(index: &mut Index, rx: &Receiver) { + let first = rx.recv_timeout(Duration::from_secs(10)).expect("an update"); + let mut pending = vec![first]; + while let Ok(next) = rx.recv_timeout(Duration::from_millis(300)) { + pending.push(next); + } + for update in pending { + match update { + Update::Files { files, live } => { + assert!(live); + for file in files { + index.insert(file); + } + } + Update::Removed { path } => { + index.remove(&path); + } + other => panic!("unexpected {other:?}"), + } + } +} + +fn write(root: &Path, rel: &str, text: &str) { + let path = root.join(rel); + fs::create_dir_all(path.parent().unwrap()).unwrap(); + fs::write(path, text).unwrap(); +} + +#[test] +fn builds_with_the_project_rules() { + let dir = tempfile::tempdir().unwrap(); + let project = lutra_testkit::generate_php_project(dir.path(), 400); + write( + dir.path(), + "src/Broken.php", + " 0); + } +} + +#[test] +fn spawned_build_ends_with_ready_and_cancels() { + let dir = tempfile::tempdir().unwrap(); + let project = lutra_testkit::generate_php_project(dir.path(), 300); + let (tx, rx) = channel(); + spawn_build(dir.path().to_path_buf(), 2, CancelToken::new(), tx).unwrap(); + let mut files = 0; + let summary = loop { + match rx.recv_timeout(Duration::from_secs(30)).unwrap() { + Update::Files { files: batch, .. } => files += batch.len(), + Update::Ready(summary) => break summary, + _ => {} + } + }; + assert_eq!(files, project.indexed_files); + assert_eq!(summary.files, files); + assert!(rx.recv_timeout(Duration::from_millis(200)).is_err()); + + let cancel = CancelToken::new(); + cancel.cancel(); + let (tx, rx) = channel(); + spawn_build(dir.path().to_path_buf(), 2, cancel, tx).unwrap(); + let summary = loop { + if let Update::Ready(summary) = rx.recv_timeout(Duration::from_secs(30)).unwrap() { + break summary; + } + }; + assert!(summary.cancelled); + assert!(summary.files < project.indexed_files); +} + +#[test] +fn updater_follows_disk_and_buffers() { + let dir = tempfile::tempdir().unwrap(); + let root = dir.path(); + write(root, ".gitignore", "*.log\n/ignored/\n"); + write(root, "src/A.php", " usize| { + let mut runs: Vec = (0..7) + .map(|_| { + let t = Instant::now(); + assert!(f() > 0); + t.elapsed().as_secs_f64() * 1e3 + }) + .collect(); + runs.sort_by(f64::total_cmp); + (runs[0], runs[3]) + }; + let class = time(&|| index.find_classes("Serv12", 50).len()); + let symbol = time(&|| index.find_symbols("handle1", 50).len()); + let member = time(&|| index.find_symbols("Service12::hand", 50).len()); + let file = time(&|| index.find_files("wid12", 50).len()); + println!( + "round {round}: {threads} workers, walk {:.0} ms, first batch {:.0} ms, total {:.0} ms, {} files, {} symbols, {} parsed ({:.1} MB)", + summary.walk.as_secs_f64() * 1e3, + first_batch.unwrap().as_secs_f64() * 1e3, + summary.elapsed.as_secs_f64() * 1e3, + summary.files, + summary.symbols, + summary.parsed, + summary.parsed_bytes as f64 / 1e6, + ); + println!( + " queries (min / median of 7): goto.class {:.1} / {:.1} ms, goto.symbol {:.1} / {:.1} ms, Class::member {:.1} / {:.1} ms, goto.file {:.1} / {:.1} ms", + class.0, class.1, symbol.0, symbol.1, member.0, member.1, file.0, file.1 + ); + } +} diff --git a/crates/index/tests/properties.rs b/crates/index/tests/properties.rs new file mode 100644 index 0000000..7c01f0e --- /dev/null +++ b/crates/index/tests/properties.rs @@ -0,0 +1,88 @@ +//! Property tests: random queries and random file contents never panic, and +//! query results respect the limit and the ranking order. + +#![allow( + clippy::unwrap_used, + clippy::expect_used, + clippy::indexing_slicing, + clippy::panic +)] + +use lutra_index::{Index, IndexedFile, index_text}; +use lutra_syntax::Parser; +use proptest::prelude::*; + +/// A small index with project and library files, unicode names included. +fn sample() -> Index { + let mut parser = Parser::new().unwrap(); + let mut index = Index::new(); + let files = [ + ( + "src/Models/User.php", + "= pair[1].score); + } + } + for hit in &files { + let len: usize = hit.path.encode_utf16().count(); + for range in &hit.matches { + prop_assert!(range[0] < range[1] && range[1] <= len); + } + prop_assert!(hit.matches.windows(2).all(|w| w[0][1] < w[1][0])); + } + prop_assert!(classes.iter().all(|h| h.symbol.kind.is_class_like())); + prop_assert!(symbols.iter().all(|h| !h.symbol.kind.is_class_like())); + } + + #[test] + fn random_sources_never_panic(bytes in proptest::collection::vec(any::(), 0..2_000)) { + let mut parser = Parser::new().unwrap(); + let mut text = String::from(" PhpProject { + let mut out = PhpProject::default(); + write(root, ".gitignore", "/var/\n/vendor/\n", &mut out, false); + out.indexed_files += 1; + let src = files.saturating_mul(80) / 100; + for i in 0..src { + let module = i / 100; + let (name, text) = src_file(i, module); + let rel = format!("src/Module{module:03}/{name}.php"); + write(root, &rel, &text, &mut out, true); + if i % 50 != 7 { + out.classes += 1; + } + } + let vendor = files.saturating_mul(15) / 100; + for i in 0..vendor { + let module = i / 100; + let rel = format!("vendor/acme/framework/src/Component{module:03}/Widget{i}.php"); + let text = class_text( + &format!("Acme\\Framework\\Component{module:03}"), + &format!("Widget{i}"), + "class", + ); + write(root, &rel, &text, &mut out, true); + out.vendor_files += 1; + out.classes += 1; + } + for i in 0..files.saturating_mul(3) / 100 { + let rel = format!("templates/page{i}.html.twig"); + write( + root, + &rel, + "{% block body %}

{{ content }}

{% endblock %}\n", + &mut out, + false, + ); + out.indexed_files += 1; + } + for i in 0..files.saturating_mul(2) / 100 { + let rel = format!("config/packages/package{i}.yaml"); + write( + root, + &rel, + &format!("package{i}:\n enabled: true\n"), + &mut out, + false, + ); + out.indexed_files += 1; + } + for i in 0..3 { + let text = class_text("Cache", &format!("Cached{i}"), "class"); + fs::create_dir_all(root.join("var/cache")).unwrap(); + fs::write(root.join(format!("var/cache/Cached{i}.php")), &text).unwrap(); + fs::create_dir_all(root.join("node_modules/pkg")).unwrap(); + fs::write(root.join(format!("node_modules/pkg/Hidden{i}.php")), &text).unwrap(); + } + out +} + +/// Name and text of the `i`th source file in module `module`. +fn src_file(i: usize, module: usize) -> (String, String) { + let namespace = format!("App\\Module{module:03}"); + match i % 50 { + 7 => ( + format!("functions{i}"), + format!( + " ( + format!("Contract{i}"), + class_text(&namespace, &format!("Contract{i}"), "interface"), + ), + n if n % 20 == 5 => ( + format!("Helps{i}"), + class_text(&namespace, &format!("Helps{i}"), "trait"), + ), + n if n % 20 == 15 => ( + format!("Status{i}"), + enum_text(&namespace, &format!("Status{i}")), + ), + _ => ( + format!("Service{i}"), + class_text(&namespace, &format!("Service{i}"), "class"), + ), + } +} + +/// A class, interface or trait of about 5 KB. +fn class_text(namespace: &str, name: &str, keyword: &str) -> String { + let mut s = format!( + "items[$key])) {{\n return $this->items[$key];\n }}\n $this->items[$key] = strtoupper($name);\n return null;\n }}\n\n" + )); + } + s.push_str("}\n"); + s +} + +/// A backed enum with a few cases and a method. +fn enum_text(namespace: &str, name: &str) -> String { + format!( + "value);\n }}\n}}\n" + ) +} + +/// Writes `text` to `root/rel`, creating directories, and counts it as an +/// indexed PHP file if `php`. +fn write(root: &Path, rel: &str, text: &str, out: &mut PhpProject, php: bool) { + let path = root.join(rel); + if let Some(parent) = path.parent() { + fs::create_dir_all(parent).unwrap(); + } + fs::write(path, text).unwrap(); + if php { + out.indexed_files += 1; + out.php_files += 1; + out.php_bytes += text.len() as u64; + } +} diff --git a/docs/architecture.md b/docs/architecture.md index ed739c7..a21a988 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -1,6 +1,6 @@ # Architecture -> **Status:** M0 decisions recorded, M1 workspace, desktop bridge, tabs and editor, message catalog, engine construction, settings, themes, the recovery journal, notifications, UI commands, the palette, the settings dialog, the app icon and the syntax layer documented, plus the search backend · **Last updated:** 2026-10-16 +> **Status:** M0 decisions recorded, M1 workspace, desktop bridge, tabs and editor, message catalog, engine construction, settings, themes, the recovery journal, notifications, UI commands, the palette, the settings dialog, the app icon and the syntax layer documented, plus the search backend and the symbol index · **Last updated:** 2026-10-17 ## Overview @@ -218,6 +218,19 @@ Watched directories: the VFS watcher is non-recursive, so `project.children` wat - Engine: `search.find {query, regex?, caseSensitive?, wholeWord?, include?, exclude?, maxResults?}` (disabled without a project) snapshots the text of every dirty buffer as an overlay (searched instead of the file; only for files the walk visits), starts the search and returns `{searchId}` at once. Searches run outside the engine lock. Each search's results wait in its own channel in `State::searches`; `Engine::pump_vfs` drains them (`cmd::search::pump`) into `searchResults` events (one per file) and a final `searchDone`. Hosts already publish the pump's events, so the desktop (pump every 100 ms) and the control channel get them without new plumbing; there are at most 10 batches per second and at most 500 `searchResults` per pump (`MAX_RESULTS_PER_PUMP`), which keeps a control subscriber's queue of 4096 events from overflowing; the rest wait for the next pump. `search.cancel {searchId}` returns `{cancelled}` (false once the search ended) and the search still ends with `searchDone {cancelled: true}`. Starting a search does not cancel older ones. A search whose project is no longer open is cancelled by the next pump, its remaining results are dropped and it ends with `cancelled: true`. - Measured (release, Windows 11, 16 threads, generated 20,006-file tree from `lutra_testkit::generate_search_tree`, other builds running): common literal `function` first result 10–43 ms, capped at 10,000 matches after 112–209 ms; a rare literal (full walk, 376 files) first result 17–22 ms, total 0.66–1.43 s (the first round cold); `function` with the 100,000 cap 0.53 s. Through the engine add up to one pump interval (100 ms) before results reach clients. Live check over the control channel against the running app (debug build, 5,900-file project with a dirty buffer): `search.find` answered in 4.5 ms, first `searchResults` line after 55 ms, `searchDone` after 386 ms; vendor hits present, `var/`, `node_modules` and the binary file absent; the dirty buffer's text was found; `search.cancel` ended a search with `cancelled: true`. +### Index and go-to (M2.5) + +- `crates/index` (L2, depends on `lutra-syntax`, `lutra-text`, `ignore` and `nucleo-matcher` 0.3, MPL-2.0 as a separate crate, built without its `unicode-segmentation` feature) lists every file of the project and the declarations of its PHP files. A `Symbol` is `{name, fqn, kind, container, line, column}`: names as written (`$id` for properties), FQNs without a leading `\` (`App\Models\User`, members `App\Models\User::find`), `container` the declaring class-like FQN, the position of the name (0-based line, UTF-16 column) from the outline's `selectionRange`. Namespaces only prefix FQNs; anonymous classes and their members are skipped; at most 20,000 symbols per file. +- Which files: the walk is configured to the same rules as `crates/search` (ignore files without a git repository, `.git` and `node_modules` skipped, the root's `vendor/` walked as a second root even when gitignored and marked `library`, dot files listed, links not followed). Every file is listed for Go to File; only `.php` files up to 2 MiB (`MAX_PARSE_BYTES`) are parsed, larger ones are listed without symbols. At most 200,000 files (`MAX_FILES`); further files are left out (`Summary::capped`). +- Build: `build`/`spawn_build` walk first (parallel `ignore` walker), report non-PHP files at once, then parse PHP files on `default_threads()` workers (cores − 1, at most 32), each with its own `Parser` and one file in memory, taking files from a shared counter: project files before library files, larger files first in each group. Workers send `Update::Files` batches of 64 files (or every 100 ms) and `Update::Progress {done, total}`; the build ends with `Update::Ready(Summary)`. A panic while parsing a file (a parser bug) is caught per file: the file is listed without symbols and the worker gets a fresh parser. Reads are capped, invalid UTF-8 is replaced, a BOM dropped. +- Incremental: `Updater` is one background thread taking `Job::File {path, text?}` (re-index from a buffer's text, or re-read from disk; a file no longer found or now excluded is reported `Removed`) and `Job::Dir {path}` (drop the subtree, list it again with a walk restricted to the path, so ignore files on the way still apply). Jobs queued while it works are coalesced to the last per path, so a burst of keystrokes parses a buffer once. +- Queries: `Index` (a `BTreeMap` by path, so a directory's files drop with one range) answers `find_files`, `find_classes` and `find_symbols`. Every whitespace-separated query word must match fuzzily (fzf-style scoring, smart case, no other special syntax); queries are capped at 256 characters and 1,000 results. Files: a query without `/` (a `\` counts as `/`) is matched against the file name first with double weight, else the path; match ranges are merged UTF-16 ranges in the path. Classes: by name, by FQN when the query has a `\`. Symbols: by name; `Class::member` matches the container's short name (FQN with a `\`) and the member separately, `Class::` lists members. Order: project before library, higher score, shorter name, byte order. Indexes over 20,000 files plus symbols are scanned on `default_threads()` threads, each keeping its own best hits. +- Engine (`cmd/goto.rs`): `State::index` holds the index of the open project. `goto::sync` runs on every `Engine::pump_vfs` and before every index command: it starts the build and the updater when a project is open and the index belongs to another root (or none), drops them without a project (dropping cancels the threads; `project.close` also drops it at once), merges up to 4,096 updates, emits `indexProgress {done, total}` at most every 250 ms while building and `indexReady {files, symbols, elapsedMs}` at the end, and re-indexes buffers. The engine lock is held only for inserts and queries; walking and parsing never run under it. Updater results that arrive while the build runs mark their paths, and later build results for those paths are ignored, so a buffer edit or a disk change is never overwritten by the stale build. +- Incremental sources: `workspace::pump` hands the VFS events to `goto::note_disk_changes` (removed paths dropped at once, created or modified files re-read, created directories and rename targets rescanned); the watcher is not recursive, so only watched directories (root, listed, pinned) report changes. A buffer whose version changed is re-indexed from its text when its path is in the index (or the build still runs); while it is dirty, disk changes for it are ignored; closing it dirty re-reads the file. +- Commands: `index.status` → `{state: "idle"|"indexing", files, symbols}` (idle and zero without a project). `goto.file {query, limit?}` → `{items: [{path, score, matches: [[start, end]]}], indexing}`; `goto.class` → `{items: [{name, fqn, kind, path, line, column, library}], indexing}`; `goto.symbol` → the same plus `container` (FQN or `null`), for functions, methods, properties, constants and enum cases. `limit` defaults to 50 (at most 1,000); an empty query returns no items. The go-to commands are disabled without a project; none has a menu path or shortcut yet (M2.6 adds the popup). +- Measured (release, Windows 11, 8 cores / 16 threads, 15 workers, generated `lutra_testkit::generate_php_project(20_000)`: 20,001 files, 19,000 PHP of ~5 KB, 87.7 MB, 324,080 symbols): build 3.9–4.2 s on an idle machine (target < 5 s met), 4.6–5.7 s while other agents' builds ran; the walk takes 25–35 ms and the first batch arrives after ~30 ms, so the parse dominates (~21 MB/s over 15 workers, ~1.4 MB/s per worker). Queries on the full index, median of 7: `goto.class` 2–4 ms, `goto.symbol` 7–11 ms, `Class::member` 11–12 ms, `goto.file` 2 ms (single-threaded before the parallel scan: `goto.symbol` 72–100 ms). `cargo test --release -p lutra-index --test perf -- --ignored --nocapture` repeats it. +- Live check over the control channel against the running app (debug build, port 7825, generated 6,002-file project with gitignored `vendor/` and `var/`): `index.status` answered `indexing` right after `project.open`, a go-to query during the build returned `indexing: true`, 25 `indexProgress` events at least 251 ms apart, `indexReady` after 8.2 s (debug build: the tree-sitter C code and the workers are unoptimised) with 6,002 files and 62,000 symbols; `goto.file`, `goto.class` (vendor marked `library`), `goto.symbol` with `Class::member` answered in 5–25 ms; the gitignored `var/` class was absent; a `buffer.edit` renaming a class and a file created on disk were found within a second; `project.close` reset the status to idle. + ## Decisions (M0) | Question | Decision | Why | @@ -235,6 +248,7 @@ Watched directories: the VFS watcher is non-recursive, so `project.children` wat | Date | Change | Summary | |---|---|---| +| 2026-10-17 | minor | M2.5: symbol index (`crates/index`: parallel build, updater, fuzzy queries), `index.status`, `goto.file`/`goto.class`/`goto.symbol`, `indexProgress`/`indexReady` via the pump, timing | | 2026-10-16 | minor | M2.3: find in files backend (`crates/search`, `search.find`/`search.cancel`, `searchResults`/`searchDone` via the pump) | | 2026-10-16 | minor | M2.1: syntax layer (tree-sitter PHP parser, incremental reparse, outline, folding, size cap, timing) | | 2026-10-16 | minor | M1.15: modal rule (one modal at a time, restore dialog queued), `recoveryResolved` event | diff --git a/docs/gaps.md b/docs/gaps.md index b1a0682..6c71115 100644 --- a/docs/gaps.md +++ b/docs/gaps.md @@ -1,6 +1,6 @@ # Gaps -> **Status:** M0 closed G1; G2 under way (text buffer, engine buffers, tabs and editor in the app, recovery journal); G3 M1 part done (project tree, palette, settings dialog, themes); G4 under way (tree-sitter outline and folding library); G14 English UI catalog in place · **Last updated:** 2026-10-16 +> **Status:** M0 closed G1; G2 under way (text buffer, engine buffers, tabs and editor in the app, recovery journal); G3 M1 part done (project tree, palette, settings dialog, themes); G4 under way (tree-sitter outline and folding library); G14 English UI catalog in place · **Last updated:** 2026-10-17 Every known shortfall, ranked. This is the work list: unless the owner gives a task, take the highest open gap. Hours are agent wall-clock hours for one agent, all estimated. @@ -31,7 +31,7 @@ G11 progress: M1.7 added the crash-recovery journal: unsaved buffers are written G14 progress: M1.11 moved every UI string in `ui/src` into the English catalog `ui/src/i18n/en.json` behind `t(key, params?)`, enforced by `vue/no-bare-strings-in-template` and a catalog test. Messages the engine sends (`EngineError.message`, the slogan from `app.about`) are still English text; they need message keys with params before a second language can cover them. -G5 progress: M2.3 landed the find-in-files backend: `crates/search` (parallel walk with the project's ignore rules, literal or regex, case, whole word, globs, binary and size skips, match cap, cancel) and the engine commands `search.find` / `search.cancel` with streamed `searchResults` and `searchDone` events; dirty buffers are searched in their buffer text. First result on a generated 20k-file tree 10–43 ms (release, measured). Open in G5: the Find tool window (M2.4), replace in files, the symbol index and go-to (M2.5, M2.6). +G5 progress: M2.3 landed the find-in-files backend: `crates/search` (parallel walk with the project's ignore rules, literal or regex, case, whole word, globs, binary and size skips, match cap, cancel) and the engine commands `search.find` / `search.cancel` with streamed `searchResults` and `searchDone` events; dirty buffers are searched in their buffer text. First result on a generated 20k-file tree 10–43 ms (release, measured). M2.5 added the first symbol index (`crates/index`: every project file, PHP declarations from tree-sitter outlines, vendor as library, parallel build off the engine lock, incremental on disk and buffer changes, cancelled on project close) and the commands `index.status`, `goto.file`, `goto.class`, `goto.symbol` with `indexProgress`/`indexReady` events; a generated 20k-file project (88 MB of PHP) indexes in 3.9–4.2 s (release, measured, target 5 s), queries take 2–12 ms. Open in G5: the Find tool window (M2.4), replace in files, the go-to popup (M2.6), an index that survives restarts, changes in directories the non-recursive watcher does not watch, usages. Known smaller items from M0: @@ -43,6 +43,7 @@ Known smaller items from M0: | Date | Change | Summary | |---|---|---| +| 2026-10-17 | minor | G5: symbol index and go-to commands (M2.5) | | 2026-10-16 | minor | G5: find-in-files backend (M2.3) | | 2026-10-16 | minor | G4 note: tree-sitter syntax library, outline and folding (M2.1) | | 2026-10-16 | minor | G3: settings dialog, theme picker (M1.9b) | diff --git a/docs/target-app-parity.md b/docs/target-app-parity.md index 703824f..3bce70f 100644 --- a/docs/target-app-parity.md +++ b/docs/target-app-parity.md @@ -1,6 +1,6 @@ # Target-app parity -> **Status:** M1 done (text buffer, VFS, engine workspace, project tree, tabs and a plain-text editor, recovery, palette, settings and themes in the app), no PHP feature ships · **Last updated:** 2026-10-16 · **Target:** PhpStorm 2026.x +> **Status:** M1 done (text buffer, VFS, engine workspace, project tree, tabs and a plain-text editor, recovery, palette, settings and themes in the app), no PHP feature ships · **Last updated:** 2026-10-17 · **Target:** PhpStorm 2026.x ## How this is measured @@ -14,7 +14,7 @@ Hours are the estimated agent wall-clock hours for one agent to reach behaviour |---|---:|---:|---|---:|---| | Editor core | 12 | 9% (est.) | syntax highlighting (M2), code folding, soft wrap and other editor settings, preview tab, pinned/split/reordered tabs, close others/all, reopen closed tab, tab limit, conflict UI for files changed on disk, autosave, local history, encodings other than UTF-8, large-file mode, column selection and multi-caret checked against the docs; recovery journal not yet checked in the running app (gaming mode) | ~106 (est.) | M2.1: `crates/syntax` parses PHP with tree-sitter (incremental reparse from `lutra_text` edits) and computes folding ranges (brace blocks, arrays, comments, heredocs, `use` runs); library only, nothing visible in the editor yet, so the score is unchanged. Tests: outline/folding snapshots for five PHP 8.3 fixtures, proptests (random bytes and token soups never panic, reparse equals a fresh parse, undo round trip). 1 MB parse 364–553 ms, reparse after one keystroke 7–11 ms (release, measured, machine busy). M1.13: opening a file that is already open switches to its tab (`bufferFocused`; engine test, Vitest, control-channel test; live check pending (gaming mode)); files in folders the tree never expanded still reload or report changes on disk, because the parent directory of every open buffer stays watched (engine tests). M1.7: crash-recovery journal (`crates/engine/src/journal.rs`, `cmd/recovery.rs`): dirty buffers are written atomically to `/recovery//` about 1 s after the last edit (at least every 2 s while typing), never per keystroke; removed on save, close and discard; `recoveryAvailable` after `project.open`, `recovery.list/restore/discard`; restored buffers are dirty and keep line ending and BOM; corrupt entries are moved aside with a warning. Restore dialog (`ui/src/components/recovery/`: Restore all / selected / Discard, time, changed-on-disk hint). Tests: engine dropped mid-edit and a new engine restores the exact CRLF + BOM text, corrupt entries, removal paths, a property test of random edit sequences → crash → restore equals the buffer (`crates/engine/tests/recovery.rs`, 10 tests), the same flow headless over the control channel (`crates/control/tests/recovery.rs`), 10 Vitest tests for the dialog. 1 MiB entry write: 8.5 ms median (release, measured). Graded against `plan/phpstorm/04-recovery.md`. M1.6: tabs and a CodeMirror 6 editor in the app (`ui/src/components/editor/`, `ui/src/stores/editor.ts`): open from the project tree or engine events, activate, close by button, middle click or `Mod+W`, dirty dot, Save / Don't Save / Cancel on closing a dirty tab, `Mod+S` / `Mod+Alt+S` from engine command specs, non-blocking notice for a dirty file changed or deleted on disk; local edits go to `buffer.edit` with version-based resync, CRLF files keep `\r\n`; undo/redo, multiple selections, search and bracket matching from CodeMirror's basic setup. Vitest: a burst of 400 keystrokes with stale versions and remote edits loses none, a CRLF burst with deletions converges, store and component tests against a fake engine (`ui/tests/editor/`, 22 tests). Checked in the app on Windows 11 with `--control` and `ui.screenshot`: typing via the UI and via `buffer.edit`, save to disk, close question, middle click, disk-change notice. 1 MB file (16,385 lines): engine `file.open` 80 ms over the control channel (debug build, measured), editor state 1–6 ms and first rendered view 4–11 ms in Chromium (measured), ≈ 90 ms in total (estimated, budget 150 ms). Graded against `plan/phpstorm/03-editor-tabs.md`. `crates/text`: rope buffer, positions, edits (M1.1, unit tests + proptests). M1.3: engine buffers with versions and dirty state, `file.open`, `buffer.get`, `buffer.edit` (UTF-16 change sets, `stale_version`), `file.save`, `file.saveAll`, `buffer.close` with unsaved-changes guard, line ending and BOM kept on save, clean buffers reload on external change; 14 integration tests on temp dirs (`crates/engine/tests/workspace.rs`) | | Project view, VFS | 6 | 8% (est.) | views list (project / scopes), toolbar (Select Opened File, Expand/Collapse All), sort and display options, show/hide excluded, distinct icons for library and excluded folders, file colours, speed search, preview, file operations | ~37 (est.) | `lutra-vfs` (M1.2): lazy sorted listing, ignore rules (.gitignore, .ignore, info/exclude, global excludes, Lutra defaults), native file watching via `notify` with a polling fallback, capped UTF-8 reads, atomic writes; 34 tests on temp dirs, watcher tests run on both backends (`crates/vfs/tests`). M1.5: project tree UI (lazy listing, keyboard navigation, dimmed ignored entries, event refresh) in `ui/src/components/project/`, Vitest against a mocked engine. M1.3: `project.open`, `project.close`, `project.children` (watches listed directories, capped at 256 native / 32 polling, least recently listed dropped), `treeChanged` events from disk, `events.subscribe` on the control channel with a TCP test (`crates/control/tests/events.rs`). M1.4: the tree runs against the engine in the app: folder picker (`project.openDialog`) and `lutra-desktop --control PORT PATH` open a project, the tree lists it lazily with ignored (`var/` from `.gitignore`) and excluded (`node_modules`) entries dimmed, a file created on disk appears in an expanded folder within ~100 ms; verified with `ui.screenshot` on Windows 11 and Vitest (`ui/tests/project/`, `ui/tests/ipcEvents.test.ts`). M1.13: a directory listed only partly (more than 100,000 entries) ends with a "Listing truncated" note row (Vitest). M1.12: a folder that cannot be opened from the picker or the command line (including `unsaved_changes` when switching projects) and stopped file watching show as toasts instead of stderr (`apps/lutra-desktop/src/host.rs`, `ui/src/stores/notifications.ts`, `ui/tests/notifications/`, 19 Vitest tests; live check pending, gaming mode); score unchanged. Graded against `plan/phpstorm/02-project-view.md`: tool window, folder-first tree, dimmed ignored/excluded, keyboard navigation and live refresh exist; most toolbar, view and display options do not | -| Navigation | 10 | 0% | everything (M1.8: a command palette over commands only, no go-to-file/class/symbol yet; score unchanged) | ~120 (est.) | M2.1: file outline in `crates/syntax` (namespaces braced and unbraced, classes, interfaces, traits, enums with cases, functions, methods, properties incl. promoted ones, constants; UTF-16 ranges and short signatures), the base for the Structure window (M2.7) and the symbol index (M2.5); not reachable from the UI yet, score unchanged. M1.8: `ui/src/components/palette/`, `ui/src/commands/` (fuzzy ranking, recent commands, global keymap), Vitest `ui/tests/palette/`, `ui/tests/commands/` | +| Navigation | 10 | 0% | everything a user can reach: the go-to popup (M2.6), Structure (M2.7), find usages, go to declaration, recent files, navigation history (M1.8: a command palette over commands only; M2.5: go-to backend without UI; score unchanged) | ~112 (est.) | M2.5: symbol index in `crates/index` (every project file listed, `.php` parsed in parallel from tree-sitter outlines, vendor marked library, incremental on disk and buffer changes, capped at 200,000 files) and the engine commands `index.status`, `goto.file`, `goto.class`, `goto.symbol` (fuzzy, file names weighted, `Class::member`, project before library, partial results with `indexing: true`), events `indexProgress`/`indexReady`; 20k-file / 88 MB generated project indexed in 3.9–4.2 s (release, measured), queries 2–12 ms; tests `crates/index/tests/`, `crates/engine/tests/goto.rs`, `crates/control/tests/goto.rs`; not reachable from the UI yet (M2.6). M2.1: file outline in `crates/syntax` (namespaces braced and unbraced, classes, interfaces, traits, enums with cases, functions, methods, properties incl. promoted ones, constants; UTF-16 ranges and short signatures), the base for the Structure window (M2.7) and the symbol index (M2.5); not reachable from the UI yet, score unchanged. M1.8: `ui/src/components/palette/`, `ui/src/commands/` (fuzzy ranking, recent commands, global keymap), Vitest `ui/tests/palette/`, `ui/tests/commands/` | | Search and replace | 6 | 0% (est.) | everything a user sees: Find in Files tool window (M2.4), replace in files, scopes, file masks UI, search history, find in path from the tree, structural search; the backend below has no UI yet, so the score stays 0% | ~76 (est.) | M2.3 backend: `crates/search` (parallel `ignore` walk + `grep-searcher`: literal/regex, case, whole word, include/exclude globs, project ignore rules with `vendor/` searched and `node_modules` skipped, binary and > 8 MiB files skipped, match cap, cancel) and engine `search.find` / `search.cancel` with `searchResults` / `searchDone` events through the pump; dirty buffers are searched in their buffer text. Tests: 9 library tests incl. a property test of random queries and file bytes (`crates/search/tests/search.rs`), 3 preview/column unit tests, 4 engine tests (`crates/engine/tests/search.rs`), a TCP control-channel test (find → results → done, invalid regex, cancel; `crates/control/tests/search.rs`). 20k-file generated tree, release: first result 10–43 ms, full walk 0.66–1.43 s (measured); live over the control channel against the running app: first result 55 ms, done 386 ms on a 5,900-file project (debug build, measured) | | Code completion | 10 | 0% | everything | ~250 (est.) | — | | Inspections and quick-fixes | 8 | 0% | everything | ~300 (est.) | — | @@ -63,6 +63,7 @@ The project tree is the first user-visible feature (Project view 8%, estimated); | Date | Change | Summary | |---|---|---| +| 2026-10-17 | minor | M2.5: Navigation evidence (symbol index and go-to commands, no UI), score unchanged at 0%, hours 120 → 112 | | 2026-10-16 | minor | M2.3: Search and replace evidence (find-in-files backend, no UI), score unchanged at 0%, hours 80 → 76 | | 2026-10-16 | minor | M2.1: Editor core and Navigation evidence (tree-sitter parse, outline, folding library), scores unchanged | | 2026-10-16 | minor | M1.9b: Settings row 4% → 7% (est.), settings dialog; M1 done |