PM: merge M2.5 symbol index and goto commands
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> # Conflicts: # docs/architecture.md # docs/gaps.md # docs/target-app-parity.md
This commit is contained in:
commit
62f9ab1564
26 changed files with 3455 additions and 9 deletions
19
Cargo.lock
generated
19
Cargo.lock
generated
|
|
@ -2582,6 +2582,7 @@ dependencies = [
|
|||
name = "lutra-engine"
|
||||
version = "0.0.0"
|
||||
dependencies = [
|
||||
"lutra-index",
|
||||
"lutra-search",
|
||||
"lutra-testkit",
|
||||
"lutra-text",
|
||||
|
|
@ -2596,6 +2597,15 @@ dependencies = [
|
|||
[[package]]
|
||||
name = "lutra-index"
|
||||
version = "0.0.0"
|
||||
dependencies = [
|
||||
"ignore",
|
||||
"lutra-syntax",
|
||||
"lutra-testkit",
|
||||
"lutra-text",
|
||||
"nucleo-matcher",
|
||||
"proptest",
|
||||
"tempfile",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "lutra-lsp"
|
||||
|
|
@ -2853,6 +2863,15 @@ dependencies = [
|
|||
"bitflags 2.13.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "nucleo-matcher"
|
||||
version = "0.3.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "bf33f538733d1a5a3494b836ba913207f14d9d4a1d3cd67030c5061bdd2cac85"
|
||||
dependencies = [
|
||||
"memchr",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "num-conv"
|
||||
version = "0.2.2"
|
||||
|
|
|
|||
170
crates/control/tests/goto.rs
Normal file
170
crates/control/tests/goto.rs
Normal file
|
|
@ -0,0 +1,170 @@
|
|||
//! The symbol index over a real TCP control connection: `indexProgress`
|
||||
//! and `indexReady` arrive as event lines from the pump, `index.status` and
|
||||
//! the go-to commands answer.
|
||||
|
||||
#![allow(
|
||||
clippy::unwrap_used,
|
||||
clippy::expect_used,
|
||||
clippy::indexing_slicing,
|
||||
clippy::panic
|
||||
)]
|
||||
|
||||
use std::io::{BufRead, BufReader, Write};
|
||||
use std::net::TcpStream;
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::time::Duration;
|
||||
|
||||
use lutra_testkit::EngineBackend;
|
||||
use serde_json::{Value, json};
|
||||
|
||||
/// A client that keeps responses and event lines apart.
|
||||
struct Client {
|
||||
reader: BufReader<TcpStream>,
|
||||
writer: TcpStream,
|
||||
events: Vec<Value>,
|
||||
}
|
||||
|
||||
impl Client {
|
||||
fn connect(addr: std::net::SocketAddr) -> Self {
|
||||
let stream = TcpStream::connect(addr).unwrap();
|
||||
stream
|
||||
.set_read_timeout(Some(Duration::from_secs(30)))
|
||||
.unwrap();
|
||||
Self {
|
||||
writer: stream.try_clone().unwrap(),
|
||||
reader: BufReader::new(stream),
|
||||
events: Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
fn line(&mut self) -> Value {
|
||||
let mut line = String::new();
|
||||
self.reader.read_line(&mut line).expect("line within 30 s");
|
||||
serde_json::from_str(&line).unwrap()
|
||||
}
|
||||
|
||||
fn call(&mut self, request: Value) -> Value {
|
||||
self.writer
|
||||
.write_all(format!("{request}\n").as_bytes())
|
||||
.unwrap();
|
||||
loop {
|
||||
let line = self.line();
|
||||
if line.get("event").is_some() {
|
||||
self.events.push(line["event"].clone());
|
||||
} else {
|
||||
return line;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Reads event lines until `indexReady`; returns the `indexProgress`
|
||||
/// events and the ready event.
|
||||
fn until_ready(&mut self) -> (Vec<Value>, Value) {
|
||||
let mut progress = Vec::new();
|
||||
let mut pending = std::mem::take(&mut self.events).into_iter();
|
||||
loop {
|
||||
let event = match pending.next() {
|
||||
Some(e) => e,
|
||||
None => {
|
||||
let line = self.line();
|
||||
let Some(event) = line.get("event") else {
|
||||
continue;
|
||||
};
|
||||
event.clone()
|
||||
}
|
||||
};
|
||||
match event["kind"].as_str() {
|
||||
Some("indexProgress") => progress.push(event),
|
||||
Some("indexReady") => return (progress, event),
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn execute(command: &str, params: Value) -> Value {
|
||||
json!({"id": command, "method": "engine.execute", "params": {"command": command, "params": params}})
|
||||
}
|
||||
|
||||
/// Pumps the backend every 20 ms, like the desktop's pump thread, until
|
||||
/// dropped.
|
||||
struct Pump(Arc<AtomicBool>);
|
||||
|
||||
impl Pump {
|
||||
fn start(backend: Arc<EngineBackend>) -> Self {
|
||||
let stop = Arc::new(AtomicBool::new(false));
|
||||
let flag = stop.clone();
|
||||
std::thread::spawn(move || {
|
||||
while !flag.load(Ordering::Relaxed) {
|
||||
backend.pump();
|
||||
std::thread::sleep(Duration::from_millis(20));
|
||||
}
|
||||
});
|
||||
Self(stop)
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for Pump {
|
||||
fn drop(&mut self) {
|
||||
self.0.store(true, Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn index_builds_and_go_to_answers_over_tcp() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let project = lutra_testkit::generate_php_project(dir.path(), 2_000);
|
||||
let backend = Arc::new(EngineBackend::new());
|
||||
let server = lutra_control::start(0, backend.clone()).unwrap();
|
||||
let mut client = Client::connect(server.local_addr());
|
||||
client.call(json!({"id": 1, "method": "events.subscribe"}));
|
||||
let r = client.call(execute("project.open", json!({"path": dir.path()})));
|
||||
assert!(r["result"]["root"].is_string(), "{r}");
|
||||
let _pump = Pump::start(backend.clone());
|
||||
|
||||
let (progress, ready) = client.until_ready();
|
||||
for event in &progress {
|
||||
assert_eq!(event["total"], project.indexed_files, "{event}");
|
||||
assert!(event["done"].as_u64().unwrap() <= project.indexed_files as u64);
|
||||
}
|
||||
assert_eq!(ready["files"], project.indexed_files);
|
||||
assert!(ready["symbols"].as_u64().unwrap() > project.classes as u64);
|
||||
assert!(ready["elapsedMs"].is_u64());
|
||||
|
||||
let r = client.call(execute("index.status", Value::Null));
|
||||
assert_eq!(r["result"]["state"], "idle", "{r}");
|
||||
assert_eq!(r["result"]["files"], project.indexed_files);
|
||||
|
||||
let r = client.call(execute(
|
||||
"goto.class",
|
||||
json!({"query": "Service10", "limit": 5}),
|
||||
));
|
||||
let items = r["result"]["items"].as_array().unwrap();
|
||||
assert_eq!(r["result"]["indexing"], false);
|
||||
assert_eq!(items[0]["fqn"], "App\\Module000\\Service10", "{r}");
|
||||
assert_eq!(items[0]["path"], "src/Module000/Service10.php");
|
||||
assert_eq!(items[0]["kind"], "class");
|
||||
assert_eq!(items[0]["library"], false);
|
||||
|
||||
let r = client.call(execute("goto.class", json!({"query": "Widget1"})));
|
||||
let first = &r["result"]["items"][0];
|
||||
assert_eq!(first["library"], true, "{r}");
|
||||
assert!(first["path"].as_str().unwrap().starts_with("vendor/"));
|
||||
|
||||
let r = client.call(execute(
|
||||
"goto.symbol",
|
||||
json!({"query": "Service10::handle3"}),
|
||||
));
|
||||
let items = r["result"]["items"].as_array().unwrap();
|
||||
assert_eq!(items[0]["fqn"], "App\\Module000\\Service10::handle3", "{r}");
|
||||
assert_eq!(items[0]["container"], "App\\Module000\\Service10");
|
||||
|
||||
let r = client.call(execute("goto.file", json!({"query": "service10.php"})));
|
||||
let first = &r["result"]["items"][0];
|
||||
assert_eq!(first["path"], "src/Module000/Service10.php", "{r}");
|
||||
assert!(first["matches"].is_array());
|
||||
|
||||
let r = client.call(execute("goto.file", json!({"query": 7})));
|
||||
assert_eq!(r["error"]["code"], "invalid_params", "{r}");
|
||||
}
|
||||
|
|
@ -12,6 +12,7 @@ publish.workspace = true
|
|||
layer = 4
|
||||
|
||||
[dependencies]
|
||||
lutra-index.workspace = true
|
||||
lutra-search.workspace = true
|
||||
lutra-text.workspace = true
|
||||
lutra-vfs.workspace = true
|
||||
|
|
|
|||
511
crates/engine/src/cmd/goto.rs
Normal file
511
crates/engine/src/cmd/goto.rs
Normal file
|
|
@ -0,0 +1,511 @@
|
|||
//! The symbol index and the go-to commands: `index.status`, `goto.file`,
|
||||
//! `goto.class` and `goto.symbol`.
|
||||
//!
|
||||
//! # Index lifecycle
|
||||
//! [`sync`] runs on every [`crate::Engine::pump_vfs`] and before every
|
||||
//! index command. When a project is open and no index exists for its root,
|
||||
//! it starts one: [`lutra_index::spawn_build`] walks and parses the project
|
||||
//! on background threads, and an [`Updater`] thread handles later changes.
|
||||
//! Both send [`Update`]s over one channel; [`sync`] merges them into the
|
||||
//! in-memory [`Index`] in [`State`], so the engine lock is only held for
|
||||
//! cheap inserts, never while files are walked or parsed. Progress becomes
|
||||
//! [`Event::IndexProgress`] (at most every [`PROGRESS_EVERY`]) and the end
|
||||
//! of the build [`Event::IndexReady`]. Closing the project (or opening
|
||||
//! another one) drops the index and cancels its threads.
|
||||
//!
|
||||
//! # Incremental updates
|
||||
//! - Disk: [`note_disk_changes`] turns the workspace's VFS events into
|
||||
//! updater jobs (created or modified files are re-read, created or
|
||||
//! renamed directories rescanned) and drops removed paths at once. The
|
||||
//! project watcher is not recursive (see [`crate::workspace`]), so changes
|
||||
//! in directories that no client listed and no buffer pins are not seen
|
||||
//! until the next project open.
|
||||
//! - Buffers: when an open buffer's version changes, its text is
|
||||
//! re-indexed instead of the file on disk; when it is closed with unsaved
|
||||
//! changes, the file on disk is re-read. Only files the index already
|
||||
//! lists (or, while building, may still list) are re-indexed from
|
||||
//! buffers, so a buffer of an ignored file never enters the index.
|
||||
//! - Updates from the updater win over build results for the same path
|
||||
//! that arrive later.
|
||||
//!
|
||||
//! # Queries
|
||||
//! Queries run on the current index, never wait for the build, and report
|
||||
//! `indexing: true` while it runs. See [`lutra_index::Index`] for matching
|
||||
//! and ranking.
|
||||
|
||||
use std::collections::{BTreeMap, HashSet};
|
||||
use std::path::PathBuf;
|
||||
use std::sync::mpsc::{Receiver, TryRecvError, channel};
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use lutra_index::{CancelToken, Index, Job, SymbolHit, Update, Updater};
|
||||
use lutra_vfs::{EntryKind, VfsEvent};
|
||||
use serde::Deserialize;
|
||||
use serde_json::{Value, json};
|
||||
|
||||
use crate::buffers::BufferId;
|
||||
use crate::cmd::params::parse;
|
||||
use crate::workspace::path_string;
|
||||
use crate::{Command, CommandSpec, EngineError, Event, Outcome, State};
|
||||
|
||||
/// Shortest time between two [`Event::IndexProgress`].
|
||||
pub const PROGRESS_EVERY: Duration = Duration::from_millis(250);
|
||||
|
||||
/// Results of a go-to query when the params give no `limit`.
|
||||
pub const DEFAULT_LIMIT: usize = 50;
|
||||
|
||||
/// Most updates merged by one [`sync`]; the rest wait for the next one, so
|
||||
/// a pump never holds the engine lock for long.
|
||||
pub const MAX_UPDATES_PER_SYNC: usize = 4_096;
|
||||
|
||||
/// The index of the open project, if one is being built or ready.
|
||||
#[derive(Debug, Default)]
|
||||
pub struct IndexState {
|
||||
/// The index and its background threads.
|
||||
running: Option<Running>,
|
||||
}
|
||||
|
||||
impl IndexState {
|
||||
/// Whether the initial build is still running.
|
||||
pub fn indexing(&self) -> bool {
|
||||
self.running.as_ref().is_some_and(|r| r.building)
|
||||
}
|
||||
|
||||
/// The index, if a project is open.
|
||||
pub fn index(&self) -> Option<&Index> {
|
||||
self.running.as_ref().map(|r| &r.index)
|
||||
}
|
||||
}
|
||||
|
||||
/// The index of one project root and its background work.
|
||||
#[derive(Debug)]
|
||||
struct Running {
|
||||
/// The project root.
|
||||
root: PathBuf,
|
||||
/// The files and symbols known so far.
|
||||
index: Index,
|
||||
/// Stops the build and the updater.
|
||||
cancel: CancelToken,
|
||||
/// Updates from the build and the updater.
|
||||
updates: Receiver<Update>,
|
||||
/// Re-indexes changed files; `None` if its thread could not start.
|
||||
updater: Option<Updater>,
|
||||
/// The initial build has not ended yet.
|
||||
building: bool,
|
||||
/// When the build started.
|
||||
started: Instant,
|
||||
/// The latest progress not yet emitted.
|
||||
progress: Option<(usize, usize)>,
|
||||
/// When progress was last emitted.
|
||||
progress_at: Option<Instant>,
|
||||
/// Paths the updater reported while the build runs; later build results
|
||||
/// for them are stale.
|
||||
touched: HashSet<String>,
|
||||
/// The path and version each open buffer was last seen at.
|
||||
buffers: BTreeMap<BufferId, (String, u64)>,
|
||||
/// Paths whose index entry comes from an unsaved buffer, so disk
|
||||
/// changes don't replace it.
|
||||
overlay: HashSet<String>,
|
||||
}
|
||||
|
||||
impl Drop for Running {
|
||||
/// Cancels the build and the updater; their threads end at the next
|
||||
/// file.
|
||||
fn drop(&mut self) {
|
||||
self.cancel.cancel();
|
||||
}
|
||||
}
|
||||
|
||||
impl Running {
|
||||
/// Starts indexing `root`. Threads that cannot start leave an index that
|
||||
/// stays empty (build) or is not updated (updater); queries still work.
|
||||
fn start(root: PathBuf) -> Self {
|
||||
let (tx, updates) = channel();
|
||||
let cancel = CancelToken::new();
|
||||
let building = lutra_index::spawn_build(
|
||||
root.clone(),
|
||||
lutra_index::default_threads(),
|
||||
cancel.clone(),
|
||||
tx.clone(),
|
||||
)
|
||||
.is_ok();
|
||||
let updater = Updater::spawn(root.clone(), cancel.clone(), tx).ok();
|
||||
Self {
|
||||
root,
|
||||
index: Index::new(),
|
||||
cancel,
|
||||
updates,
|
||||
updater,
|
||||
building,
|
||||
started: Instant::now(),
|
||||
progress: None,
|
||||
progress_at: None,
|
||||
touched: HashSet::new(),
|
||||
buffers: BTreeMap::new(),
|
||||
overlay: HashSet::new(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Queues `job` for the updater.
|
||||
fn send(&self, job: Job) {
|
||||
if let Some(updater) = &self.updater {
|
||||
updater.send(job);
|
||||
}
|
||||
}
|
||||
|
||||
/// Merges `update` into the index; returns the event it causes.
|
||||
fn apply(&mut self, update: Update) -> Option<Event> {
|
||||
match update {
|
||||
Update::Files { files, live } => {
|
||||
for file in files {
|
||||
if live {
|
||||
if self.building {
|
||||
self.touched.insert(file.path.clone());
|
||||
}
|
||||
} else if self.touched.contains(&file.path) {
|
||||
continue;
|
||||
}
|
||||
self.index.insert(file);
|
||||
}
|
||||
None
|
||||
}
|
||||
Update::Removed { path } => {
|
||||
self.index.remove(&path);
|
||||
if self.building {
|
||||
self.touched.insert(path);
|
||||
}
|
||||
None
|
||||
}
|
||||
Update::Progress { done, total } => {
|
||||
self.progress = Some((done, total));
|
||||
None
|
||||
}
|
||||
Update::Ready(summary) => Some(self.ready(summary.elapsed)),
|
||||
}
|
||||
}
|
||||
|
||||
/// Ends the build after `elapsed` and returns [`Event::IndexReady`].
|
||||
fn ready(&mut self, elapsed: Duration) -> Event {
|
||||
self.building = false;
|
||||
self.progress = None;
|
||||
self.touched.clear();
|
||||
Event::IndexReady {
|
||||
files: self.index.len(),
|
||||
symbols: self.index.symbol_count(),
|
||||
elapsed_ms: u64::try_from(elapsed.as_millis()).unwrap_or(u64::MAX),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Brings the index in step with the state: starts it for a newly open
|
||||
/// project, drops it without one, merges finished updates, re-indexes
|
||||
/// changed buffers and emits progress and readiness events. Called by the
|
||||
/// pump and by every index command.
|
||||
pub fn sync(state: &mut State) {
|
||||
let root = state
|
||||
.project
|
||||
.as_ref()
|
||||
.map(|w| w.project.root().to_path_buf());
|
||||
let Some(root) = root else {
|
||||
state.index.running = None;
|
||||
return;
|
||||
};
|
||||
if state.index.running.as_ref().is_none_or(|r| r.root != root) {
|
||||
state.index.running = Some(Running::start(root));
|
||||
}
|
||||
let Some(running) = state.index.running.as_mut() else {
|
||||
return;
|
||||
};
|
||||
let mut events = Vec::new();
|
||||
for _ in 0..MAX_UPDATES_PER_SYNC {
|
||||
match running.updates.try_recv() {
|
||||
Ok(update) => events.extend(running.apply(update)),
|
||||
Err(TryRecvError::Empty) => break,
|
||||
Err(TryRecvError::Disconnected) => {
|
||||
if running.building {
|
||||
let elapsed = running.started.elapsed();
|
||||
events.push(running.ready(elapsed));
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
if let Some((done, total)) = running.progress
|
||||
&& running.building
|
||||
&& running
|
||||
.progress_at
|
||||
.is_none_or(|at| at.elapsed() >= PROGRESS_EVERY)
|
||||
{
|
||||
running.progress = None;
|
||||
running.progress_at = Some(Instant::now());
|
||||
events.push(Event::IndexProgress { done, total });
|
||||
}
|
||||
sync_buffers(state);
|
||||
for event in events {
|
||||
state.emit(event);
|
||||
}
|
||||
}
|
||||
|
||||
/// Re-indexes buffers whose text changed since the last sync, and re-reads
|
||||
/// the files of closed buffers that were indexed from unsaved text.
|
||||
fn sync_buffers(state: &mut State) {
|
||||
let Some(running) = state.index.running.as_mut() else {
|
||||
return;
|
||||
};
|
||||
let mut seen = BTreeMap::new();
|
||||
for id in state.buffers.ids() {
|
||||
let Some(open) = state.buffers.get(id) else {
|
||||
continue;
|
||||
};
|
||||
let path = path_string(&open.path);
|
||||
let dirty = open.dirty();
|
||||
let changed = match running.buffers.get(&id) {
|
||||
Some((old_path, version)) => *version != open.version || *old_path != path,
|
||||
None => dirty,
|
||||
};
|
||||
let known = running.index.contains(&path) || running.building;
|
||||
if changed && known {
|
||||
if dirty {
|
||||
running.overlay.insert(path.clone());
|
||||
}
|
||||
running.send(Job::File {
|
||||
path: path.clone(),
|
||||
text: Some(open.buffer.clone()),
|
||||
});
|
||||
}
|
||||
if !dirty {
|
||||
running.overlay.remove(&path);
|
||||
}
|
||||
seen.insert(id, (path, open.version));
|
||||
}
|
||||
let closed: Vec<String> = running
|
||||
.buffers
|
||||
.iter()
|
||||
.filter(|(id, _)| !seen.contains_key(id))
|
||||
.map(|(_, (path, _))| path.clone())
|
||||
.collect();
|
||||
for path in closed {
|
||||
if running.overlay.remove(&path) {
|
||||
running.send(Job::File { path, text: None });
|
||||
}
|
||||
}
|
||||
running.buffers = seen;
|
||||
}
|
||||
|
||||
/// Turns changes on disk into index updates: removed paths are dropped at
|
||||
/// once, created and modified files are re-read, created directories and
|
||||
/// rename targets are rescanned. Files indexed from an unsaved buffer keep
|
||||
/// the buffer's text. Does nothing without an index.
|
||||
pub fn note_disk_changes(state: &mut State, changes: &[VfsEvent]) {
|
||||
let Some(running) = state.index.running.as_mut() else {
|
||||
return;
|
||||
};
|
||||
for change in changes {
|
||||
match change {
|
||||
VfsEvent::Created { path, kind } => {
|
||||
let path = path_string(path);
|
||||
if *kind == EntryKind::Directory {
|
||||
running.send(Job::Dir { path });
|
||||
} else if !running.overlay.contains(&path) {
|
||||
running.send(Job::File { path, text: None });
|
||||
}
|
||||
}
|
||||
VfsEvent::Modified { path } => {
|
||||
let path = path_string(path);
|
||||
if !running.overlay.contains(&path) {
|
||||
running.send(Job::File { path, text: None });
|
||||
}
|
||||
}
|
||||
VfsEvent::Removed { path } => {
|
||||
let path = path_string(path);
|
||||
running.apply(Update::Removed { path });
|
||||
}
|
||||
VfsEvent::Renamed { from, to } => {
|
||||
running.apply(Update::Removed {
|
||||
path: path_string(from),
|
||||
});
|
||||
running.send(Job::Dir {
|
||||
path: path_string(to),
|
||||
});
|
||||
}
|
||||
VfsEvent::Failed(_) => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Drops the index and cancels its threads; `project.close` calls it.
|
||||
pub fn close(state: &mut State) {
|
||||
state.index.running = None;
|
||||
}
|
||||
|
||||
/// `index.status`: whether the index is building and how big it is.
|
||||
pub struct Status;
|
||||
|
||||
impl Command for Status {
|
||||
fn spec(&self) -> &CommandSpec {
|
||||
&CommandSpec {
|
||||
id: "index.status",
|
||||
label: "Index Status",
|
||||
menu_path: None,
|
||||
default_shortcut: None,
|
||||
params: "none; returns { state: \"idle\" | \"indexing\", files, symbols }; events indexProgress { done, total } (at most every 250 ms) and indexReady { files, symbols, elapsedMs } report the build after project.open",
|
||||
}
|
||||
}
|
||||
|
||||
fn run(&self, state: &mut State, _params: Value) -> Result<Outcome, EngineError> {
|
||||
sync(state);
|
||||
let index = state.index.index();
|
||||
Ok(Outcome::value(json!({
|
||||
"state": if state.index.indexing() { "indexing" } else { "idle" },
|
||||
"files": index.map_or(0, Index::len),
|
||||
"symbols": index.map_or(0, Index::symbol_count),
|
||||
})))
|
||||
}
|
||||
}
|
||||
|
||||
/// Params of the go-to commands.
|
||||
#[derive(Deserialize)]
|
||||
#[serde(deny_unknown_fields, rename_all = "camelCase")]
|
||||
struct QueryParams {
|
||||
/// What to look for.
|
||||
query: String,
|
||||
/// Most results.
|
||||
limit: Option<usize>,
|
||||
}
|
||||
|
||||
/// Parses the params of the go-to command `id`, brings the index up to
|
||||
/// date and returns the query and the limit.
|
||||
fn prepare(state: &mut State, id: &str, params: Value) -> Result<(String, usize), EngineError> {
|
||||
let p: QueryParams = parse(id, params)?;
|
||||
sync(state);
|
||||
Ok((p.query, p.limit.unwrap_or(DEFAULT_LIMIT)))
|
||||
}
|
||||
|
||||
/// The spec of a go-to command.
|
||||
const fn goto_spec(id: &'static str, label: &'static str, params: &'static str) -> CommandSpec {
|
||||
CommandSpec {
|
||||
id,
|
||||
label,
|
||||
menu_path: None,
|
||||
default_shortcut: None,
|
||||
params,
|
||||
}
|
||||
}
|
||||
|
||||
/// `goto.file`: fuzzy search over the project's file paths.
|
||||
pub struct File;
|
||||
|
||||
impl Command for File {
|
||||
fn spec(&self) -> &CommandSpec {
|
||||
const SPEC: CommandSpec = goto_spec(
|
||||
"goto.file",
|
||||
"Go to File",
|
||||
"{ query, limit? (default 50, at most 1000) }; returns { items: [{ path, score, matches: [[start, end]] }], indexing }: fuzzy over project-relative paths, file names weighted higher, project files before vendor files; matches are UTF-16 ranges in path; indexing is true while the index is still being built (partial results)",
|
||||
);
|
||||
&SPEC
|
||||
}
|
||||
|
||||
fn enabled(&self, state: &State) -> bool {
|
||||
state.has_project()
|
||||
}
|
||||
|
||||
fn run(&self, state: &mut State, params: Value) -> Result<Outcome, EngineError> {
|
||||
let (query, limit) = prepare(state, self.spec().id, params)?;
|
||||
let items: Vec<Value> = state
|
||||
.index
|
||||
.index()
|
||||
.map(|index| index.find_files(&query, limit))
|
||||
.unwrap_or_default()
|
||||
.into_iter()
|
||||
.map(|hit| json!({ "path": hit.path, "score": hit.score, "matches": hit.matches }))
|
||||
.collect();
|
||||
Ok(Outcome::value(
|
||||
json!({ "items": items, "indexing": state.index.indexing() }),
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
/// `goto.class`: fuzzy search over classes, interfaces, traits and enums.
|
||||
pub struct Class;
|
||||
|
||||
impl Command for Class {
|
||||
fn spec(&self) -> &CommandSpec {
|
||||
const SPEC: CommandSpec = goto_spec(
|
||||
"goto.class",
|
||||
"Go to Class",
|
||||
"{ query, limit? (default 50, at most 1000) }; returns { items: [{ name, fqn, kind, path, line, column, library }], indexing }: classes, interfaces, traits and enums by name (by FQN when the query has a backslash), project before library; line and column (UTF-16) point at the name",
|
||||
);
|
||||
&SPEC
|
||||
}
|
||||
|
||||
fn enabled(&self, state: &State) -> bool {
|
||||
state.has_project()
|
||||
}
|
||||
|
||||
fn run(&self, state: &mut State, params: Value) -> Result<Outcome, EngineError> {
|
||||
let (query, limit) = prepare(state, self.spec().id, params)?;
|
||||
let items: Vec<Value> = state
|
||||
.index
|
||||
.index()
|
||||
.map(|index| index.find_classes(&query, limit))
|
||||
.unwrap_or_default()
|
||||
.iter()
|
||||
.map(|hit| symbol_item(hit, false))
|
||||
.collect();
|
||||
Ok(Outcome::value(
|
||||
json!({ "items": items, "indexing": state.index.indexing() }),
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
/// `goto.symbol`: fuzzy search over functions, methods, properties,
|
||||
/// constants and enum cases.
|
||||
pub struct Symbol;
|
||||
|
||||
impl Command for Symbol {
|
||||
fn spec(&self) -> &CommandSpec {
|
||||
const SPEC: CommandSpec = goto_spec(
|
||||
"goto.symbol",
|
||||
"Go to Symbol",
|
||||
"{ query, limit? (default 50, at most 1000) }; returns { items: [{ name, fqn, kind, container, path, line, column, library }], indexing }: functions, methods, properties, constants and enum cases by name; Class::member matches the class and the member; container is the declaring class FQN or null",
|
||||
);
|
||||
&SPEC
|
||||
}
|
||||
|
||||
fn enabled(&self, state: &State) -> bool {
|
||||
state.has_project()
|
||||
}
|
||||
|
||||
fn run(&self, state: &mut State, params: Value) -> Result<Outcome, EngineError> {
|
||||
let (query, limit) = prepare(state, self.spec().id, params)?;
|
||||
let items: Vec<Value> = state
|
||||
.index
|
||||
.index()
|
||||
.map(|index| index.find_symbols(&query, limit))
|
||||
.unwrap_or_default()
|
||||
.iter()
|
||||
.map(|hit| symbol_item(hit, true))
|
||||
.collect();
|
||||
Ok(Outcome::value(
|
||||
json!({ "items": items, "indexing": state.index.indexing() }),
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
/// The JSON item of a class or symbol hit, with `container` for symbols.
|
||||
fn symbol_item(hit: &SymbolHit<'_>, container: bool) -> Value {
|
||||
let s = hit.symbol;
|
||||
let mut item = json!({
|
||||
"name": s.name,
|
||||
"fqn": s.fqn,
|
||||
"kind": s.kind.as_str(),
|
||||
"path": hit.path,
|
||||
"line": s.line,
|
||||
"column": s.column,
|
||||
"library": hit.library,
|
||||
});
|
||||
if container && let Some(map) = item.as_object_mut() {
|
||||
map.insert("container".into(), json!(s.container));
|
||||
}
|
||||
item
|
||||
}
|
||||
|
|
@ -3,6 +3,7 @@
|
|||
mod app;
|
||||
mod buffer;
|
||||
mod file;
|
||||
pub mod goto;
|
||||
mod params;
|
||||
mod project;
|
||||
mod recovery;
|
||||
|
|
@ -42,5 +43,9 @@ pub fn register_builtins(registry: &mut Registry) -> Result<(), EngineError> {
|
|||
registry.register(Box::new(ui::RegisterCommands))?;
|
||||
registry.register(Box::new(search::Find))?;
|
||||
registry.register(Box::new(search::Cancel))?;
|
||||
registry.register(Box::new(goto::Status))?;
|
||||
registry.register(Box::new(goto::File))?;
|
||||
registry.register(Box::new(goto::Class))?;
|
||||
registry.register(Box::new(goto::Symbol))?;
|
||||
Ok(())
|
||||
}
|
||||
|
|
|
|||
|
|
@ -58,6 +58,7 @@ pub fn close_project(state: &mut State, force: bool) -> Result<(), EngineError>
|
|||
}
|
||||
state.journal = None;
|
||||
state.project = None;
|
||||
crate::cmd::goto::close(state);
|
||||
state.emit(Event::ProjectClosed);
|
||||
Ok(())
|
||||
}
|
||||
|
|
|
|||
|
|
@ -131,7 +131,8 @@ impl Engine {
|
|||
/// Applies pending changes on disk (reloads clean buffers, reports
|
||||
/// changed directories and conflicts), writes due recovery-journal
|
||||
/// entries (see [`crate::journal`]), moves finished search results into
|
||||
/// events (see [`cmd::search`]) and returns every queued event.
|
||||
/// events (see [`cmd::search`]), merges index updates (see
|
||||
/// [`cmd::goto`]) and returns every queued event.
|
||||
/// Hosts call it regularly, e.g. every 100 ms from a background thread.
|
||||
///
|
||||
/// # Errors
|
||||
|
|
@ -143,6 +144,7 @@ impl Engine {
|
|||
crate::workspace::pump(state);
|
||||
crate::journal::tick(state, std::time::Instant::now());
|
||||
cmd::search::pump(state);
|
||||
cmd::goto::sync(state);
|
||||
Ok(())
|
||||
})?;
|
||||
Ok(state.take_events())
|
||||
|
|
@ -368,6 +370,10 @@ mod tests {
|
|||
"file.open",
|
||||
"file.save",
|
||||
"file.saveAll",
|
||||
"goto.class",
|
||||
"goto.file",
|
||||
"goto.symbol",
|
||||
"index.status",
|
||||
"project.children",
|
||||
"project.close",
|
||||
"project.open",
|
||||
|
|
|
|||
|
|
@ -169,6 +169,24 @@ pub enum Event {
|
|||
/// Wall time of the search in milliseconds.
|
||||
elapsed_ms: u64,
|
||||
},
|
||||
/// The symbol index is being built after `project.open`; sent at most
|
||||
/// every 250 ms while it runs.
|
||||
IndexProgress {
|
||||
/// Files indexed so far.
|
||||
done: usize,
|
||||
/// Files found in the project.
|
||||
total: usize,
|
||||
},
|
||||
/// The initial build of the symbol index ended (or was cut short); go-to
|
||||
/// queries now see the whole project.
|
||||
IndexReady {
|
||||
/// Files in the index.
|
||||
files: usize,
|
||||
/// Symbols in the index.
|
||||
symbols: usize,
|
||||
/// Wall time of the build in milliseconds.
|
||||
elapsed_ms: u64,
|
||||
},
|
||||
}
|
||||
|
||||
/// One match of [`Event::SearchResults`]. Positions are 0-based; columns
|
||||
|
|
@ -312,6 +330,27 @@ mod tests {
|
|||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn index_shapes() {
|
||||
let e = Event::IndexProgress {
|
||||
done: 10,
|
||||
total: 20,
|
||||
};
|
||||
assert_eq!(
|
||||
serde_json::to_value(e).unwrap(),
|
||||
json!({"kind": "indexProgress", "done": 10, "total": 20})
|
||||
);
|
||||
let e = Event::IndexReady {
|
||||
files: 20,
|
||||
symbols: 300,
|
||||
elapsed_ms: 1500,
|
||||
};
|
||||
assert_eq!(
|
||||
serde_json::to_value(e).unwrap(),
|
||||
json!({"kind": "indexReady", "files": 20, "symbols": 300, "elapsedMs": 1500})
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn recovery_resolved_shape() {
|
||||
let e = Event::RecoveryResolved {
|
||||
|
|
|
|||
|
|
@ -37,6 +37,8 @@ pub struct State {
|
|||
pub ui_commands: BTreeMap<String, UiCommand>,
|
||||
/// Running searches (`search.find`).
|
||||
pub searches: crate::cmd::search::Searches,
|
||||
/// The symbol index of the open project (`index.status`, `goto.*`).
|
||||
pub index: crate::cmd::goto::IndexState,
|
||||
/// Number of notifications emitted, for their ids.
|
||||
notifications: u64,
|
||||
/// Events queued for the host, oldest first.
|
||||
|
|
|
|||
|
|
@ -234,7 +234,8 @@ pub fn sync_buffer_dirs(state: &mut State) {
|
|||
workspace.pin_dirs(dirs);
|
||||
}
|
||||
|
||||
/// Applies pending changes on disk to the state: queues
|
||||
/// Applies pending changes on disk to the state: hands them to the symbol
|
||||
/// index ([`crate::cmd::goto::note_disk_changes`]), queues
|
||||
/// [`Event::TreeChanged`] for affected directories, reloads clean buffers
|
||||
/// whose file changed, and queues [`Event::FileChangedOnDisk`] for dirty
|
||||
/// buffers and for buffers whose file disappeared.
|
||||
|
|
@ -244,6 +245,7 @@ pub fn pump(state: &mut State) {
|
|||
};
|
||||
let (changes, failure) = workspace.drain();
|
||||
let root = workspace.project.root().to_path_buf();
|
||||
crate::cmd::goto::note_disk_changes(state, &changes);
|
||||
let mut dirs: Vec<PathBuf> = Vec::new();
|
||||
let mut add_dir = |path: &Path| {
|
||||
let dir = path.parent().map(Path::to_path_buf).unwrap_or_default();
|
||||
|
|
|
|||
336
crates/engine/tests/goto.rs
Normal file
336
crates/engine/tests/goto.rs
Normal file
|
|
@ -0,0 +1,336 @@
|
|||
//! The symbol index in the engine: `index.status`, the go-to commands,
|
||||
//! progress and readiness events, and incremental updates from disk and
|
||||
//! buffers.
|
||||
|
||||
#![allow(
|
||||
clippy::unwrap_used,
|
||||
clippy::expect_used,
|
||||
clippy::panic,
|
||||
clippy::indexing_slicing
|
||||
)]
|
||||
|
||||
use std::fs;
|
||||
use std::path::Path;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use lutra_engine::{Engine, Event};
|
||||
use proptest::prelude::*;
|
||||
use serde_json::{Value, json};
|
||||
use tempfile::TempDir;
|
||||
|
||||
fn write(root: &Path, rel: &str, text: &str) {
|
||||
let abs = root.join(rel);
|
||||
fs::create_dir_all(abs.parent().unwrap()).unwrap();
|
||||
fs::write(abs, text).unwrap();
|
||||
}
|
||||
|
||||
/// A temp folder with `files`, opened in a fresh engine.
|
||||
fn open_project(files: &[(&str, &str)]) -> (TempDir, Engine) {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
for (path, text) in files {
|
||||
write(dir.path(), path, text);
|
||||
}
|
||||
let mut engine = Engine::new().unwrap();
|
||||
engine
|
||||
.execute("project.open", json!({"path": dir.path()}))
|
||||
.unwrap();
|
||||
(dir, engine)
|
||||
}
|
||||
|
||||
fn run(engine: &mut Engine, command: &str, params: Value) -> Value {
|
||||
engine.execute(command, params).unwrap().value
|
||||
}
|
||||
|
||||
/// Pumps until `indexReady`; returns every event seen.
|
||||
fn until_ready(engine: &mut Engine) -> Vec<Event> {
|
||||
let deadline = Instant::now() + Duration::from_secs(30);
|
||||
let mut seen = Vec::new();
|
||||
while Instant::now() < deadline {
|
||||
let events = engine.pump_vfs().unwrap();
|
||||
let ready = events.iter().any(|e| matches!(e, Event::IndexReady { .. }));
|
||||
seen.extend(events);
|
||||
if ready {
|
||||
return seen;
|
||||
}
|
||||
std::thread::sleep(Duration::from_millis(10));
|
||||
}
|
||||
panic!("no indexReady within 30 s");
|
||||
}
|
||||
|
||||
/// Pumps until `query` with `command` gives `count` items.
|
||||
fn until_count(engine: &mut Engine, command: &str, query: &str, count: usize) -> Value {
|
||||
let deadline = Instant::now() + Duration::from_secs(10);
|
||||
loop {
|
||||
engine.pump_vfs().unwrap();
|
||||
let out = run(engine, command, json!({"query": query}));
|
||||
if out["items"].as_array().unwrap().len() == count {
|
||||
return out;
|
||||
}
|
||||
assert!(
|
||||
Instant::now() < deadline,
|
||||
"{command} {query}: wanted {count} items, got {out}"
|
||||
);
|
||||
std::thread::sleep(Duration::from_millis(20));
|
||||
}
|
||||
}
|
||||
|
||||
fn names(out: &Value, field: &str) -> Vec<String> {
|
||||
out["items"]
|
||||
.as_array()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.map(|i| i[field].as_str().unwrap().to_string())
|
||||
.collect()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn commands_need_a_project_and_status_is_idle_without_one() {
|
||||
let mut engine = Engine::new().unwrap();
|
||||
assert_eq!(
|
||||
run(&mut engine, "index.status", Value::Null),
|
||||
json!({"state": "idle", "files": 0, "symbols": 0})
|
||||
);
|
||||
for command in ["goto.file", "goto.class", "goto.symbol"] {
|
||||
let err = engine.execute(command, json!({"query": "a"})).unwrap_err();
|
||||
assert_eq!(err.code(), "disabled");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn builds_in_the_background_and_answers_go_to_queries() {
|
||||
let (_dir, mut engine) = open_project(&[
|
||||
(".gitignore", "/var/\n/vendor/\n"),
|
||||
(
|
||||
"src/Models/User.php",
|
||||
"<?php\nnamespace App\\Models;\n\nfinal class User\n{\n public const ROLE = 'user';\n private ?string $email = null;\n public function find(int $id): ?self { return null; }\n}\n",
|
||||
),
|
||||
(
|
||||
"src/helpers.php",
|
||||
"<?php\nfunction format_user(): string { return ''; }\n",
|
||||
),
|
||||
("templates/user.html.twig", "{{ user }}"),
|
||||
("var/cache/UserCache.php", "<?php class UserCache {}"),
|
||||
(
|
||||
"vendor/acme/lib/User.php",
|
||||
"<?php\nnamespace Acme;\n\ninterface User { public function find(); }\n",
|
||||
),
|
||||
]);
|
||||
let events = until_ready(&mut engine);
|
||||
let ready = events
|
||||
.iter()
|
||||
.find_map(|e| match e {
|
||||
Event::IndexReady { files, symbols, .. } => Some((*files, *symbols)),
|
||||
_ => None,
|
||||
})
|
||||
.unwrap();
|
||||
assert_eq!(ready, (5, 7));
|
||||
assert_eq!(
|
||||
run(&mut engine, "index.status", Value::Null),
|
||||
json!({"state": "idle", "files": 5, "symbols": 7})
|
||||
);
|
||||
|
||||
let files = run(&mut engine, "goto.file", json!({"query": "user"}));
|
||||
assert_eq!(files["indexing"], false);
|
||||
assert_eq!(
|
||||
names(&files, "path"),
|
||||
vec![
|
||||
"src/Models/User.php",
|
||||
"templates/user.html.twig",
|
||||
"vendor/acme/lib/User.php",
|
||||
]
|
||||
);
|
||||
assert_eq!(files["items"][0]["matches"], json!([[11, 15]]));
|
||||
assert!(files["items"][0]["score"].as_u64().unwrap() > 0);
|
||||
|
||||
let classes = run(&mut engine, "goto.class", json!({"query": "user"}));
|
||||
assert_eq!(
|
||||
classes["items"],
|
||||
json!([
|
||||
{"name": "User", "fqn": "App\\Models\\User", "kind": "class", "path": "src/Models/User.php", "line": 3, "column": 12, "library": false},
|
||||
{"name": "User", "fqn": "Acme\\User", "kind": "interface", "path": "vendor/acme/lib/User.php", "line": 3, "column": 10, "library": true},
|
||||
])
|
||||
);
|
||||
|
||||
let symbols = run(
|
||||
&mut engine,
|
||||
"goto.symbol",
|
||||
json!({"query": "find", "limit": 1}),
|
||||
);
|
||||
assert_eq!(
|
||||
symbols["items"],
|
||||
json!([{"name": "find", "fqn": "App\\Models\\User::find", "kind": "method", "container": "App\\Models\\User", "path": "src/Models/User.php", "line": 7, "column": 20, "library": false}])
|
||||
);
|
||||
let function = run(&mut engine, "goto.symbol", json!({"query": "format_user"}));
|
||||
assert_eq!(function["items"][0]["container"], Value::Null);
|
||||
let members = run(&mut engine, "goto.symbol", json!({"query": "User::"}));
|
||||
assert_eq!(
|
||||
names(&members, "name"),
|
||||
vec!["ROLE", "find", "$email", "find"]
|
||||
);
|
||||
let empty = run(&mut engine, "goto.class", json!({"query": ""}));
|
||||
assert_eq!(empty["items"], json!([]));
|
||||
|
||||
for bad in [
|
||||
json!({}),
|
||||
json!({"query": 3}),
|
||||
json!({"query": "a", "limit": -1}),
|
||||
json!({"query": "a", "extra": true}),
|
||||
] {
|
||||
let err = engine.execute("goto.file", bad).unwrap_err();
|
||||
assert_eq!(err.code(), "invalid_params");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reports_progress_and_partial_results_while_indexing() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let project = lutra_testkit::generate_php_project(dir.path(), 3_000);
|
||||
let mut engine = Engine::new().unwrap();
|
||||
engine
|
||||
.execute("project.open", json!({"path": dir.path()}))
|
||||
.unwrap();
|
||||
let status = run(&mut engine, "index.status", Value::Null);
|
||||
assert_eq!(status["state"], "indexing");
|
||||
let early = run(&mut engine, "goto.class", json!({"query": "Service"}));
|
||||
assert_eq!(early["indexing"], true);
|
||||
let events = until_ready(&mut engine);
|
||||
let progress: Vec<(usize, usize)> = events
|
||||
.iter()
|
||||
.filter_map(|e| match e {
|
||||
Event::IndexProgress { done, total } => Some((*done, *total)),
|
||||
_ => None,
|
||||
})
|
||||
.collect();
|
||||
assert!(
|
||||
progress
|
||||
.iter()
|
||||
.all(|(_, total)| *total == project.indexed_files)
|
||||
);
|
||||
assert!(progress.windows(2).all(|w| w[0].0 <= w[1].0));
|
||||
let status = run(&mut engine, "index.status", Value::Null);
|
||||
assert_eq!(status["state"], "idle");
|
||||
assert_eq!(status["files"], project.indexed_files);
|
||||
let classes = run(
|
||||
&mut engine,
|
||||
"goto.class",
|
||||
json!({"query": "Service1", "limit": 1000}),
|
||||
);
|
||||
assert_eq!(classes["indexing"], false);
|
||||
assert!(!classes["items"].as_array().unwrap().is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn follows_buffers_and_disk_changes() {
|
||||
let (dir, mut engine) = open_project(&[("src/A.php", "<?php class Alpha {}")]);
|
||||
until_ready(&mut engine);
|
||||
run(&mut engine, "project.children", json!({"dir": "src"}));
|
||||
|
||||
let opened = run(&mut engine, "file.open", json!({"path": "src/A.php"}));
|
||||
let id = opened["id"].as_u64().unwrap();
|
||||
let version = opened["version"].as_u64().unwrap();
|
||||
run(
|
||||
&mut engine,
|
||||
"buffer.edit",
|
||||
json!({"id": id, "baseVersion": version, "edits": [{"from": 12, "to": 17, "insert": "Omega"}]}),
|
||||
);
|
||||
until_count(&mut engine, "goto.class", "Omega", 1);
|
||||
until_count(&mut engine, "goto.class", "Alpha", 0);
|
||||
|
||||
write(dir.path(), "src/A.php", "<?php class Disk {}");
|
||||
std::thread::sleep(Duration::from_millis(300));
|
||||
for _ in 0..10 {
|
||||
engine.pump_vfs().unwrap();
|
||||
}
|
||||
assert_eq!(
|
||||
names(
|
||||
&run(&mut engine, "goto.class", json!({"query": "Omega"})),
|
||||
"name"
|
||||
),
|
||||
vec!["Omega"],
|
||||
"the unsaved buffer wins over the disk"
|
||||
);
|
||||
run(
|
||||
&mut engine,
|
||||
"buffer.close",
|
||||
json!({"id": id, "force": true}),
|
||||
);
|
||||
until_count(&mut engine, "goto.class", "Disk", 1);
|
||||
until_count(&mut engine, "goto.class", "Omega", 0);
|
||||
|
||||
write(dir.path(), "src/B.php", "<?php class Beta {}");
|
||||
until_count(&mut engine, "goto.class", "Beta", 1);
|
||||
fs::remove_file(dir.path().join("src/B.php")).unwrap();
|
||||
until_count(&mut engine, "goto.class", "Beta", 0);
|
||||
until_count(&mut engine, "goto.file", "B.php", 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn closing_the_project_drops_the_index() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
lutra_testkit::generate_php_project(dir.path(), 2_000);
|
||||
let mut engine = Engine::new().unwrap();
|
||||
engine
|
||||
.execute("project.open", json!({"path": dir.path()}))
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
run(&mut engine, "index.status", Value::Null)["state"],
|
||||
"indexing"
|
||||
);
|
||||
run(&mut engine, "project.close", json!({}));
|
||||
assert_eq!(
|
||||
run(&mut engine, "index.status", Value::Null),
|
||||
json!({"state": "idle", "files": 0, "symbols": 0})
|
||||
);
|
||||
for _ in 0..5 {
|
||||
let events = engine.pump_vfs().unwrap();
|
||||
assert!(
|
||||
!events
|
||||
.iter()
|
||||
.any(|e| matches!(e, Event::IndexProgress { .. } | Event::IndexReady { .. }))
|
||||
);
|
||||
}
|
||||
|
||||
let (other, mut engine2) = open_project(&[("a.php", "<?php class First {}")]);
|
||||
until_ready(&mut engine2);
|
||||
let second = tempfile::tempdir().unwrap();
|
||||
write(second.path(), "b.php", "<?php class Second {}");
|
||||
engine2
|
||||
.execute(
|
||||
"project.open",
|
||||
json!({"path": second.path(), "force": true}),
|
||||
)
|
||||
.unwrap();
|
||||
until_ready(&mut engine2);
|
||||
assert!(
|
||||
run(&mut engine2, "goto.class", json!({"query": "First"}))["items"]
|
||||
.as_array()
|
||||
.unwrap()
|
||||
.is_empty()
|
||||
);
|
||||
assert_eq!(
|
||||
run(&mut engine2, "goto.class", json!({"query": "Second"}))["items"]
|
||||
.as_array()
|
||||
.unwrap()
|
||||
.len(),
|
||||
1
|
||||
);
|
||||
drop(other);
|
||||
}
|
||||
|
||||
proptest! {
|
||||
#![proptest_config(ProptestConfig::with_cases(64))]
|
||||
|
||||
#[test]
|
||||
fn random_go_to_params_never_panic(query in "\\PC{0,64}", limit in proptest::option::of(0usize..5_000)) {
|
||||
let (_dir, mut engine) = open_project(&[("src/A.php", "<?php namespace N; class Alpha { function go() {} }")]);
|
||||
for command in ["goto.file", "goto.class", "goto.symbol"] {
|
||||
let mut params = json!({"query": query});
|
||||
if let Some(limit) = limit {
|
||||
params["limit"] = json!(limit);
|
||||
}
|
||||
let out = engine.execute(command, params).unwrap().value;
|
||||
let items = out["items"].as_array().unwrap();
|
||||
prop_assert!(items.len() <= limit.unwrap_or(50).min(1_000));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -12,6 +12,15 @@ publish.workspace = true
|
|||
layer = 2
|
||||
|
||||
[dependencies]
|
||||
ignore.workspace = true
|
||||
lutra-syntax.workspace = true
|
||||
lutra-text.workspace = true
|
||||
nucleo-matcher = { version = "0.3.1", default-features = false, features = ["unicode-normalization", "unicode-casefold"] }
|
||||
|
||||
[dev-dependencies]
|
||||
lutra-testkit.workspace = true
|
||||
proptest.workspace = true
|
||||
tempfile.workspace = true
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
|
|
|
|||
257
crates/index/src/build.rs
Normal file
257
crates/index/src/build.rs
Normal file
|
|
@ -0,0 +1,257 @@
|
|||
//! Building the index of a whole project in parallel.
|
||||
|
||||
use std::io::Read;
|
||||
use std::panic::{AssertUnwindSafe, catch_unwind};
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering};
|
||||
use std::sync::mpsc::Sender;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use lutra_syntax::Parser;
|
||||
|
||||
use crate::symbol::{is_library, is_php};
|
||||
use crate::walk::{Found, walk};
|
||||
use crate::{
|
||||
BATCH_FILES, CancelToken, IndexedFile, MAX_FILES, MAX_PARSE_BYTES, Summary, Update, index_text,
|
||||
};
|
||||
|
||||
/// Longest a worker keeps indexed files before sending them.
|
||||
const FLUSH_AFTER: Duration = Duration::from_millis(100);
|
||||
|
||||
/// Files a worker collects before sending them.
|
||||
const WORKER_BATCH: usize = 64;
|
||||
|
||||
/// Walks the project at `root` and indexes it, reporting [`Update::Files`]
|
||||
/// batches and [`Update::Progress`] through `send` while it runs (from the
|
||||
/// calling thread and the workers). Blocks until every file is indexed or
|
||||
/// `cancel` fires. Does not send [`Update::Ready`]; the caller turns the
|
||||
/// returned summary into it.
|
||||
///
|
||||
/// Files without symbols (not PHP, or larger than [`MAX_PARSE_BYTES`]) are
|
||||
/// reported right after the walk. PHP files are then parsed on `threads`
|
||||
/// workers, project files before library files and larger files first
|
||||
/// within each group, so the longest parses start early. A file that
|
||||
/// cannot be read is listed without symbols.
|
||||
pub fn build(
|
||||
root: &Path,
|
||||
threads: usize,
|
||||
cancel: &CancelToken,
|
||||
send: &(dyn Fn(Update) + Sync),
|
||||
) -> Summary {
|
||||
let started = Instant::now();
|
||||
let walked = walk(root, None, threads, MAX_FILES, cancel);
|
||||
let walk_time = started.elapsed();
|
||||
let total = walked.files.len();
|
||||
let (mut php, rest): (Vec<Found>, Vec<Found>) = walked
|
||||
.files
|
||||
.into_iter()
|
||||
.partition(|f| is_php(&f.path) && f.size <= MAX_PARSE_BYTES);
|
||||
let mut done = 0usize;
|
||||
let mut batch = Vec::with_capacity(BATCH_FILES);
|
||||
for found in rest {
|
||||
batch.push(IndexedFile {
|
||||
library: is_library(&found.path),
|
||||
path: found.path,
|
||||
symbols: Vec::new(),
|
||||
});
|
||||
if batch.len() >= BATCH_FILES {
|
||||
done = done.saturating_add(batch.len());
|
||||
send(Update::Files {
|
||||
files: std::mem::take(&mut batch),
|
||||
live: false,
|
||||
});
|
||||
}
|
||||
}
|
||||
done = done.saturating_add(batch.len());
|
||||
if !batch.is_empty() {
|
||||
send(Update::Files {
|
||||
files: batch,
|
||||
live: false,
|
||||
});
|
||||
}
|
||||
send(Update::Progress { done, total });
|
||||
php.sort_by(|a, b| {
|
||||
is_library(&a.path)
|
||||
.cmp(&is_library(&b.path))
|
||||
.then(b.size.cmp(&a.size))
|
||||
});
|
||||
let shared = Shared {
|
||||
files: &php,
|
||||
next: AtomicUsize::new(0),
|
||||
done: AtomicUsize::new(done),
|
||||
total,
|
||||
symbols: AtomicUsize::new(0),
|
||||
parsed: AtomicUsize::new(0),
|
||||
parsed_bytes: AtomicU64::new(0),
|
||||
cancel,
|
||||
send,
|
||||
};
|
||||
let workers = threads.clamp(1, php.len().max(1));
|
||||
std::thread::scope(|scope| {
|
||||
for _ in 0..workers {
|
||||
scope.spawn(|| shared.work());
|
||||
}
|
||||
});
|
||||
Summary {
|
||||
files: shared.done.load(Ordering::Relaxed),
|
||||
symbols: shared.symbols.load(Ordering::Relaxed),
|
||||
parsed: shared.parsed.load(Ordering::Relaxed),
|
||||
parsed_bytes: shared.parsed_bytes.load(Ordering::Relaxed),
|
||||
capped: walked.capped,
|
||||
cancelled: cancel.is_cancelled(),
|
||||
walk: walk_time,
|
||||
elapsed: started.elapsed(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Runs [`build`] on a thread of its own and sends its updates, then
|
||||
/// [`Update::Ready`], over `updates`. Updates sent after the receiver is
|
||||
/// gone are dropped; the caller cancels the build when it stops listening.
|
||||
///
|
||||
/// # Errors
|
||||
/// The thread could not be started.
|
||||
pub fn spawn_build(
|
||||
root: PathBuf,
|
||||
threads: usize,
|
||||
cancel: CancelToken,
|
||||
updates: Sender<Update>,
|
||||
) -> std::io::Result<()> {
|
||||
std::thread::Builder::new()
|
||||
.name("lutra-index".into())
|
||||
.spawn(move || {
|
||||
let send = |update| {
|
||||
let _ = updates.send(update);
|
||||
};
|
||||
let summary = build(&root, threads, &cancel, &send);
|
||||
send(Update::Ready(summary));
|
||||
})
|
||||
.map(|_| ())
|
||||
}
|
||||
|
||||
/// State the workers of one build share.
|
||||
struct Shared<'a> {
|
||||
/// PHP files to parse, in order.
|
||||
files: &'a [Found],
|
||||
/// Index of the next file to take.
|
||||
next: AtomicUsize,
|
||||
/// Files indexed and sent.
|
||||
done: AtomicUsize,
|
||||
/// Files found by the walk.
|
||||
total: usize,
|
||||
/// Symbols found.
|
||||
symbols: AtomicUsize,
|
||||
/// Files parsed.
|
||||
parsed: AtomicUsize,
|
||||
/// Bytes parsed.
|
||||
parsed_bytes: AtomicU64,
|
||||
/// Stops the workers.
|
||||
cancel: &'a CancelToken,
|
||||
/// Receives the batches.
|
||||
send: &'a (dyn Fn(Update) + Sync),
|
||||
}
|
||||
|
||||
impl Shared<'_> {
|
||||
/// One worker: takes files until none are left or the build is
|
||||
/// cancelled, and sends them in batches.
|
||||
fn work(&self) {
|
||||
let mut parser = Parser::new().ok();
|
||||
let mut batch = Vec::with_capacity(WORKER_BATCH);
|
||||
let mut last = Instant::now();
|
||||
while !self.cancel.is_cancelled() {
|
||||
let i = self.next.fetch_add(1, Ordering::Relaxed);
|
||||
let Some(found) = self.files.get(i) else {
|
||||
break;
|
||||
};
|
||||
let (file, bytes) = index_file(&mut parser, found);
|
||||
if let Some(bytes) = bytes {
|
||||
self.parsed.fetch_add(1, Ordering::Relaxed);
|
||||
self.parsed_bytes.fetch_add(bytes, Ordering::Relaxed);
|
||||
}
|
||||
self.symbols
|
||||
.fetch_add(file.symbols.len(), Ordering::Relaxed);
|
||||
batch.push(file);
|
||||
if batch.len() >= WORKER_BATCH || last.elapsed() >= FLUSH_AFTER {
|
||||
self.flush(&mut batch);
|
||||
last = Instant::now();
|
||||
}
|
||||
}
|
||||
self.flush(&mut batch);
|
||||
}
|
||||
|
||||
/// Sends `batch` and the progress after it.
|
||||
fn flush(&self, batch: &mut Vec<IndexedFile>) {
|
||||
if batch.is_empty() {
|
||||
return;
|
||||
}
|
||||
let count = batch.len();
|
||||
(self.send)(Update::Files {
|
||||
files: std::mem::take(batch),
|
||||
live: false,
|
||||
});
|
||||
let done = self
|
||||
.done
|
||||
.fetch_add(count, Ordering::Relaxed)
|
||||
.saturating_add(count);
|
||||
(self.send)(Update::Progress {
|
||||
done,
|
||||
total: self.total,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
/// Reads and indexes one file. Returns the file and the bytes parsed,
|
||||
/// `None` when it was listed without symbols: it is not a PHP file, could
|
||||
/// not be read, grew past [`MAX_PARSE_BYTES`], no parser is available, or
|
||||
/// parsing panicked.
|
||||
pub(crate) fn index_file(parser: &mut Option<Parser>, found: &Found) -> (IndexedFile, Option<u64>) {
|
||||
let empty = || IndexedFile {
|
||||
path: found.path.clone(),
|
||||
library: is_library(&found.path),
|
||||
symbols: Vec::new(),
|
||||
};
|
||||
if !is_php(&found.path) {
|
||||
return (empty(), None);
|
||||
}
|
||||
let Some(text) = read_source(&found.abs) else {
|
||||
return (empty(), None);
|
||||
};
|
||||
match guarded_index(parser, found.path.clone(), &text) {
|
||||
Some(file) => (file, Some(text.len() as u64)),
|
||||
None => (empty(), None),
|
||||
}
|
||||
}
|
||||
|
||||
/// [`index_text`] with a guard: `None` when no parser is available or
|
||||
/// parsing panicked (a bug in the parser; the parser is replaced so the
|
||||
/// next file gets a fresh one, and the index goes on without this file's
|
||||
/// symbols).
|
||||
pub(crate) fn guarded_index(
|
||||
parser: &mut Option<Parser>,
|
||||
path: String,
|
||||
text: &str,
|
||||
) -> Option<IndexedFile> {
|
||||
let p = parser.as_mut()?;
|
||||
match catch_unwind(AssertUnwindSafe(|| index_text(p, path, text))) {
|
||||
Ok(file) => Some(file),
|
||||
Err(_) => {
|
||||
*parser = Parser::new().ok();
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// The text of the source file at `path`: at most [`MAX_PARSE_BYTES`],
|
||||
/// without a UTF-8 byte order mark, invalid UTF-8 replaced. `None` if it
|
||||
/// cannot be read or is larger.
|
||||
pub(crate) fn read_source(path: &Path) -> Option<String> {
|
||||
let file = std::fs::File::open(path).ok()?;
|
||||
let mut bytes = Vec::new();
|
||||
file.take(MAX_PARSE_BYTES.saturating_add(1))
|
||||
.read_to_end(&mut bytes)
|
||||
.ok()?;
|
||||
if bytes.len() as u64 > MAX_PARSE_BYTES {
|
||||
return None;
|
||||
}
|
||||
let body = bytes.strip_prefix(b"\xEF\xBB\xBF").unwrap_or(&bytes);
|
||||
Some(String::from_utf8_lossy(body).into_owned())
|
||||
}
|
||||
|
|
@ -1,4 +1,145 @@
|
|||
//! Symbol index for Lutra (layer L2).
|
||||
//!
|
||||
//! Stores declarations and usages for go-to and find-usages, starting with
|
||||
//! symbols extracted from tree-sitter trees.
|
||||
//! Lists every file of a project and the declarations of its PHP files
|
||||
//! (classes, interfaces, traits, enums, functions, methods, properties,
|
||||
//! constants, enum cases), extracted from tree-sitter outlines
|
||||
//! ([`lutra_syntax::outline`]), and answers fuzzy go-to queries over them.
|
||||
//!
|
||||
//! # Which files
|
||||
//! The walk follows the rules of `lutra-vfs` and `lutra-search`:
|
||||
//! `.gitignore`, `.ignore`, `.git/info/exclude` and the global git excludes
|
||||
//! apply without a git repository; `.git` and every `node_modules` directory
|
||||
//! are skipped; the root's `vendor/` is walked even when an ignore file
|
||||
//! lists it and its files are marked as library code; dot files are listed;
|
||||
//! symbolic links are not followed. Every file is listed for
|
||||
//! [`Index::find_files`]; only `.php` files are parsed, and only those up to
|
||||
//! [`MAX_PARSE_BYTES`]. At most [`MAX_FILES`] files are indexed.
|
||||
//!
|
||||
//! # Building
|
||||
//! [`build`] walks the project, then parses the PHP files on a pool of
|
||||
//! worker threads (one parser each, one file in memory each), project files
|
||||
//! before library files and larger files first within each group. It
|
||||
//! reports [`Update`]s through a callback: batches of indexed files,
|
||||
//! progress, and a final [`Summary`]. [`spawn_build`] runs it on a thread of
|
||||
//! its own and sends the updates over a channel. [`Updater`] re-indexes
|
||||
//! single files and directories later, from disk or from an editor buffer.
|
||||
//! A [`CancelToken`] stops both between files.
|
||||
//!
|
||||
//! # Robustness
|
||||
//! Source files are untrusted: reads are capped, invalid UTF-8 is replaced,
|
||||
//! broken PHP yields the declarations that could be recognised, symbols per
|
||||
//! file are capped ([`MAX_SYMBOLS_PER_FILE`]), and a panic while parsing one
|
||||
//! file (a bug) loses that file's symbols, not the index.
|
||||
|
||||
mod build;
|
||||
mod query;
|
||||
mod symbol;
|
||||
mod updater;
|
||||
mod walk;
|
||||
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::time::Duration;
|
||||
|
||||
pub use build::{build, spawn_build};
|
||||
pub use query::{FileHit, Index, MAX_LIMIT, MAX_QUERY_CHARS, SymbolHit};
|
||||
pub use symbol::{IndexedFile, Symbol, SymbolKind, index_text, is_library, symbols_from_outline};
|
||||
pub use updater::{Job, Updater};
|
||||
|
||||
/// Most files one index holds.
|
||||
pub const MAX_FILES: usize = 200_000;
|
||||
|
||||
/// Largest PHP file parsed, in bytes (2 MiB). Larger files are listed for
|
||||
/// [`Index::find_files`] without symbols: tree-sitter-php parses about
|
||||
/// 2–3 MB/s, so one such file would hold a worker for a second or more, and
|
||||
/// files this size are generated code rather than declarations to navigate.
|
||||
pub const MAX_PARSE_BYTES: u64 = 2 * 1024 * 1024;
|
||||
|
||||
/// Most symbols kept from one file; the rest are dropped.
|
||||
pub const MAX_SYMBOLS_PER_FILE: usize = 20_000;
|
||||
|
||||
/// Most worker threads of one build.
|
||||
pub const MAX_THREADS: usize = 32;
|
||||
|
||||
/// Files per [`Update::Files`] batch of a build.
|
||||
pub const BATCH_FILES: usize = 256;
|
||||
|
||||
/// What a build or an [`Updater`] reports.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub enum Update {
|
||||
/// Indexed files that replace any earlier entry with the same path.
|
||||
Files {
|
||||
/// The files.
|
||||
files: Vec<IndexedFile>,
|
||||
/// From an [`Updater`] (a change after the build started), so newer
|
||||
/// than anything the build reports for the same path.
|
||||
live: bool,
|
||||
},
|
||||
/// A file or directory is gone: drop `path` and everything below it.
|
||||
Removed {
|
||||
/// Project-relative path with `/` separators.
|
||||
path: String,
|
||||
},
|
||||
/// Build progress: `done` of `total` files are indexed.
|
||||
Progress {
|
||||
/// Files indexed so far.
|
||||
done: usize,
|
||||
/// Files found by the walk.
|
||||
total: usize,
|
||||
},
|
||||
/// The build ended; nothing from it follows.
|
||||
Ready(Summary),
|
||||
}
|
||||
|
||||
/// How a build ended.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
|
||||
pub struct Summary {
|
||||
/// Files indexed.
|
||||
pub files: usize,
|
||||
/// Symbols found in them.
|
||||
pub symbols: usize,
|
||||
/// PHP files parsed.
|
||||
pub parsed: usize,
|
||||
/// Bytes of PHP parsed.
|
||||
pub parsed_bytes: u64,
|
||||
/// The walk found more than [`MAX_FILES`] files; the rest are left out.
|
||||
pub capped: bool,
|
||||
/// The build was cancelled before it finished.
|
||||
pub cancelled: bool,
|
||||
/// Wall time of the walk alone.
|
||||
pub walk: Duration,
|
||||
/// Wall time from start to end.
|
||||
pub elapsed: Duration,
|
||||
}
|
||||
|
||||
/// Stops a running build or updater. Clones share the flag.
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub struct CancelToken(Arc<AtomicBool>);
|
||||
|
||||
impl CancelToken {
|
||||
/// A token that is not cancelled.
|
||||
pub fn new() -> Self {
|
||||
Self::default()
|
||||
}
|
||||
|
||||
/// Asks the work to stop at the next file.
|
||||
pub fn cancel(&self) {
|
||||
self.0.store(true, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
/// Whether [`CancelToken::cancel`] was called.
|
||||
pub fn is_cancelled(&self) -> bool {
|
||||
self.0.load(Ordering::Relaxed)
|
||||
}
|
||||
}
|
||||
|
||||
/// Worker threads for a build on this machine: one less than the cores,
|
||||
/// so the UI and the engine keep a core, at least one and at most
|
||||
/// [`MAX_THREADS`].
|
||||
pub fn default_threads() -> usize {
|
||||
std::thread::available_parallelism()
|
||||
.map(|n| n.get())
|
||||
.unwrap_or(2)
|
||||
.saturating_sub(1)
|
||||
.clamp(1, MAX_THREADS)
|
||||
}
|
||||
|
|
|
|||
667
crates/index/src/query.rs
Normal file
667
crates/index/src/query.rs
Normal file
|
|
@ -0,0 +1,667 @@
|
|||
//! The index in memory and the fuzzy go-to queries over it.
|
||||
//!
|
||||
//! Matching uses `nucleo-matcher` (fzf-style scoring: consecutive
|
||||
//! characters, word starts and path separators score higher). The query is
|
||||
//! split at whitespace and every word must match; no other syntax is
|
||||
//! special. Lower-case query characters match either case, upper-case ones
|
||||
//! only themselves (smart case). Project files and symbols always rank
|
||||
//! before library ones; within each group the higher score wins, then the
|
||||
//! shorter name, then the name in byte order, so results are stable.
|
||||
|
||||
use std::cmp::Ordering;
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
use nucleo_matcher::pattern::{AtomKind, CaseMatching, Normalization, Pattern};
|
||||
use nucleo_matcher::{Config, Matcher, Utf32Str};
|
||||
|
||||
use crate::{IndexedFile, MAX_FILES, Symbol};
|
||||
|
||||
/// Most results one query returns.
|
||||
pub const MAX_LIMIT: usize = 1_000;
|
||||
|
||||
/// Longest query, in characters; the rest is ignored.
|
||||
pub const MAX_QUERY_CHARS: usize = 256;
|
||||
|
||||
/// Score factor of a match inside the file name over a match in the whole
|
||||
/// path, so `user` finds `User.php` before `user/Repository.php`.
|
||||
const NAME_WEIGHT: u32 = 2;
|
||||
|
||||
/// Files plus symbols from which a query is scanned on several threads.
|
||||
const PARALLEL_FROM: usize = 20_000;
|
||||
|
||||
/// The files and symbols of a project.
|
||||
#[derive(Debug, Default)]
|
||||
pub struct Index {
|
||||
/// Files by project-relative path.
|
||||
files: BTreeMap<String, IndexedFile>,
|
||||
/// Symbols in all files.
|
||||
symbols: usize,
|
||||
}
|
||||
|
||||
/// A file found by [`Index::find_files`].
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct FileHit {
|
||||
/// Project-relative path with `/` separators.
|
||||
pub path: String,
|
||||
/// Match score; higher is better.
|
||||
pub score: u32,
|
||||
/// Matched character ranges in `path`, `[start, end)` in UTF-16 code
|
||||
/// units, ascending and not overlapping.
|
||||
pub matches: Vec<[usize; 2]>,
|
||||
/// Library code (below `vendor/`).
|
||||
pub library: bool,
|
||||
}
|
||||
|
||||
/// A symbol found by [`Index::find_classes`] or [`Index::find_symbols`].
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub struct SymbolHit<'a> {
|
||||
/// The symbol.
|
||||
pub symbol: &'a Symbol,
|
||||
/// Project-relative path of its file.
|
||||
pub path: &'a str,
|
||||
/// Library code (below `vendor/`).
|
||||
pub library: bool,
|
||||
/// Match score; higher is better.
|
||||
pub score: u32,
|
||||
}
|
||||
|
||||
impl Index {
|
||||
/// An empty index.
|
||||
pub fn new() -> Self {
|
||||
Self::default()
|
||||
}
|
||||
|
||||
/// Number of files.
|
||||
pub fn len(&self) -> usize {
|
||||
self.files.len()
|
||||
}
|
||||
|
||||
/// Whether the index holds no files.
|
||||
pub fn is_empty(&self) -> bool {
|
||||
self.files.is_empty()
|
||||
}
|
||||
|
||||
/// Number of symbols in all files.
|
||||
pub fn symbol_count(&self) -> usize {
|
||||
self.symbols
|
||||
}
|
||||
|
||||
/// The file at the project-relative `path`.
|
||||
pub fn get(&self, path: &str) -> Option<&IndexedFile> {
|
||||
self.files.get(path)
|
||||
}
|
||||
|
||||
/// Whether the index holds the file at `path`.
|
||||
pub fn contains(&self, path: &str) -> bool {
|
||||
self.files.contains_key(path)
|
||||
}
|
||||
|
||||
/// Adds `file`, replacing the entry with the same path. A new file is
|
||||
/// refused (returns `false`) when the index already holds
|
||||
/// [`MAX_FILES`] files.
|
||||
pub fn insert(&mut self, file: IndexedFile) -> bool {
|
||||
if self.files.len() >= MAX_FILES && !self.files.contains_key(&file.path) {
|
||||
return false;
|
||||
}
|
||||
self.symbols = self.symbols.saturating_add(file.symbols.len());
|
||||
if let Some(old) = self.files.insert(file.path.clone(), file) {
|
||||
self.symbols = self.symbols.saturating_sub(old.symbols.len());
|
||||
}
|
||||
true
|
||||
}
|
||||
|
||||
/// Drops the file at `path`, or every file below the directory `path`.
|
||||
/// Returns the number of files dropped.
|
||||
pub fn remove(&mut self, path: &str) -> usize {
|
||||
let path = path.trim_matches('/');
|
||||
if let Some(old) = self.files.remove(path) {
|
||||
self.symbols = self.symbols.saturating_sub(old.symbols.len());
|
||||
return 1;
|
||||
}
|
||||
let keys: Vec<String> = if path.is_empty() {
|
||||
self.files.keys().cloned().collect()
|
||||
} else {
|
||||
let start = format!("{path}/");
|
||||
let end = format!("{path}0");
|
||||
self.files
|
||||
.range(start..end)
|
||||
.map(|(k, _)| k.clone())
|
||||
.collect()
|
||||
};
|
||||
for key in &keys {
|
||||
if let Some(old) = self.files.remove(key) {
|
||||
self.symbols = self.symbols.saturating_sub(old.symbols.len());
|
||||
}
|
||||
}
|
||||
keys.len()
|
||||
}
|
||||
|
||||
/// Drops everything.
|
||||
pub fn clear(&mut self) {
|
||||
self.files.clear();
|
||||
self.symbols = 0;
|
||||
}
|
||||
|
||||
/// Files whose project-relative path matches `query`, best first, at
|
||||
/// most `limit` (capped at [`MAX_LIMIT`]).
|
||||
///
|
||||
/// A query without `/` (or `\`, read as `/`) is matched against the
|
||||
/// file name first, which scores [`NAME_WEIGHT`] times higher, and
|
||||
/// against the whole path otherwise; a query with `/` against the whole
|
||||
/// path only. An empty query finds nothing.
|
||||
pub fn find_files(&self, query: &str, limit: usize) -> Vec<FileHit> {
|
||||
let query = clamp(query).replace('\\', "/");
|
||||
let by_path = query.contains('/');
|
||||
let Some(mut fuzzy) = Fuzzy::new(&query, Config::DEFAULT.match_paths()) else {
|
||||
return Vec::new();
|
||||
};
|
||||
let order = |a: &(&IndexedFile, u32, bool), b: &(&IndexedFile, u32, bool)| {
|
||||
rank((a.0.library, a.1, &a.0.path), (b.0.library, b.1, &b.0.path))
|
||||
};
|
||||
let hits = self.scan(
|
||||
limit,
|
||||
|| Fuzzy::new(&query, Config::DEFAULT.match_paths()),
|
||||
|fuzzy, file, hits| {
|
||||
let name = file_name(&file.path);
|
||||
if !by_path && let Some(score) = fuzzy.score(name) {
|
||||
hits.push((file, score.saturating_mul(NAME_WEIGHT), true));
|
||||
} else if let Some(score) = fuzzy.score(&file.path) {
|
||||
hits.push((file, score, false));
|
||||
}
|
||||
},
|
||||
order,
|
||||
);
|
||||
hits.into_iter()
|
||||
.map(|(file, score, on_name)| {
|
||||
let name = file_name(&file.path);
|
||||
let (haystack, base) = if on_name {
|
||||
let prefix = file.path.len().saturating_sub(name.len());
|
||||
let base = file.path.get(..prefix).map_or(0, utf16_len);
|
||||
(name, base)
|
||||
} else {
|
||||
(file.path.as_str(), 0)
|
||||
};
|
||||
FileHit {
|
||||
path: file.path.clone(),
|
||||
score,
|
||||
matches: ranges(haystack, &fuzzy.indices(haystack), base),
|
||||
library: file.library,
|
||||
}
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Classes, interfaces, traits and enums whose name matches `query`,
|
||||
/// best first, at most `limit` (capped at [`MAX_LIMIT`]). A query with
|
||||
/// `\` is matched against the fully qualified name instead; a leading
|
||||
/// `\` is ignored. An empty query finds nothing.
|
||||
pub fn find_classes(&self, query: &str, limit: usize) -> Vec<SymbolHit<'_>> {
|
||||
self.find_symbols_where(query, limit, true)
|
||||
}
|
||||
|
||||
/// Functions, methods, properties, constants and enum cases whose name
|
||||
/// matches `query`, best first, at most `limit` (capped at
|
||||
/// [`MAX_LIMIT`]). `Class::member` matches the container's short name
|
||||
/// (or its FQN when the part before `::` has a `\`) and the member name
|
||||
/// separately; `Class::` lists the members of matching classes. A query
|
||||
/// with `\` and no `::` is matched against the fully qualified name. An
|
||||
/// empty query finds nothing.
|
||||
pub fn find_symbols(&self, query: &str, limit: usize) -> Vec<SymbolHit<'_>> {
|
||||
self.find_symbols_where(query, limit, false)
|
||||
}
|
||||
|
||||
/// The shared part of [`Index::find_classes`] (`classes`) and
|
||||
/// [`Index::find_symbols`].
|
||||
fn find_symbols_where<'a>(
|
||||
&'a self,
|
||||
query: &str,
|
||||
limit: usize,
|
||||
classes: bool,
|
||||
) -> Vec<SymbolHit<'a>> {
|
||||
let query = clamp(query);
|
||||
let query = query.trim().trim_start_matches('\\');
|
||||
let config = symbol_config();
|
||||
let split = if classes {
|
||||
None
|
||||
} else {
|
||||
query.split_once("::")
|
||||
};
|
||||
let (owner_query, member_query) = match split {
|
||||
Some((owner, member)) => (Some(owner.trim_start_matches('\\')), member),
|
||||
None => (None, query),
|
||||
};
|
||||
let owner_by_fqn = owner_query.is_some_and(|q| q.contains('\\'));
|
||||
let by_fqn = owner_query.is_none() && member_query.contains('\\');
|
||||
let matchers = || {
|
||||
let owner = owner_query.and_then(|q| Fuzzy::new(q, config.clone()));
|
||||
let member = Fuzzy::new(member_query, config.clone());
|
||||
(owner.is_some() || member.is_some()).then_some((owner, member))
|
||||
};
|
||||
if matchers().is_none() {
|
||||
return Vec::new();
|
||||
}
|
||||
let visit = |(owner, member): &mut (Option<Fuzzy>, Option<Fuzzy>),
|
||||
file: &'a IndexedFile,
|
||||
hits: &mut Vec<SymbolHit<'a>>| {
|
||||
for symbol in &file.symbols {
|
||||
if symbol.kind.is_class_like() != classes {
|
||||
continue;
|
||||
}
|
||||
let mut score = 0u32;
|
||||
if let Some(owner) = owner.as_mut() {
|
||||
let Some(container) = symbol.container.as_deref() else {
|
||||
continue;
|
||||
};
|
||||
let haystack = if owner_by_fqn {
|
||||
container
|
||||
} else {
|
||||
container.rsplit('\\').next().unwrap_or(container)
|
||||
};
|
||||
let Some(s) = owner.score(haystack) else {
|
||||
continue;
|
||||
};
|
||||
score = s;
|
||||
}
|
||||
if let Some(member) = member.as_mut() {
|
||||
let haystack = if by_fqn { &symbol.fqn } else { &symbol.name };
|
||||
let Some(s) = member.score(haystack) else {
|
||||
continue;
|
||||
};
|
||||
score = score.saturating_add(s);
|
||||
}
|
||||
hits.push(SymbolHit {
|
||||
symbol,
|
||||
path: &file.path,
|
||||
library: file.library,
|
||||
score,
|
||||
});
|
||||
}
|
||||
};
|
||||
self.scan(limit, matchers, visit, |a, b| {
|
||||
rank(
|
||||
(a.library, a.score, &a.symbol.name),
|
||||
(b.library, b.score, &b.symbol.name),
|
||||
)
|
||||
.then_with(|| a.symbol.fqn.cmp(&b.symbol.fqn))
|
||||
.then_with(|| a.path.cmp(b.path))
|
||||
.then_with(|| a.symbol.line.cmp(&b.symbol.line))
|
||||
})
|
||||
}
|
||||
|
||||
/// Runs `visit` over every file and returns the best `limit` hits in
|
||||
/// `order`. Large indexes are scanned on [`crate::default_threads`]
|
||||
/// threads, each with its own matcher state from `make` and its own
|
||||
/// best hits, merged at the end; small ones on the calling thread.
|
||||
/// Returns nothing when `make` gives no state (an empty query).
|
||||
fn scan<'a, S, T: Send>(
|
||||
&'a self,
|
||||
limit: usize,
|
||||
make: impl Fn() -> Option<S> + Sync,
|
||||
visit: impl Fn(&mut S, &'a IndexedFile, &mut Vec<T>) + Sync,
|
||||
order: impl Fn(&T, &T) -> Ordering + Sync,
|
||||
) -> Vec<T> {
|
||||
let files: Vec<&IndexedFile> = self.files.values().collect();
|
||||
let work = files.len().saturating_add(self.symbols);
|
||||
let threads = if work < PARALLEL_FROM {
|
||||
1
|
||||
} else {
|
||||
crate::default_threads()
|
||||
};
|
||||
let chunk = files.len().div_ceil(threads).max(1);
|
||||
let run = |part: &[&'a IndexedFile]| {
|
||||
let mut hits = Vec::new();
|
||||
if let Some(mut state) = make() {
|
||||
for file in part {
|
||||
visit(&mut state, file, &mut hits);
|
||||
}
|
||||
}
|
||||
best(&mut hits, limit, &order);
|
||||
hits
|
||||
};
|
||||
let mut hits: Vec<T> = if threads <= 1 {
|
||||
run(&files)
|
||||
} else {
|
||||
std::thread::scope(|scope| {
|
||||
let handles: Vec<_> = files
|
||||
.chunks(chunk)
|
||||
.map(|part| scope.spawn(|| run(part)))
|
||||
.collect();
|
||||
handles
|
||||
.into_iter()
|
||||
.flat_map(|h| h.join().unwrap_or_default())
|
||||
.collect()
|
||||
})
|
||||
};
|
||||
best(&mut hits, limit, &order);
|
||||
hits
|
||||
}
|
||||
}
|
||||
|
||||
/// The ranking order: project before library, higher score first, then
|
||||
/// shorter, then byte order.
|
||||
fn rank(a: (bool, u32, &str), b: (bool, u32, &str)) -> Ordering {
|
||||
a.0.cmp(&b.0)
|
||||
.then_with(|| b.1.cmp(&a.1))
|
||||
.then_with(|| a.2.len().cmp(&b.2.len()))
|
||||
.then_with(|| a.2.cmp(b.2))
|
||||
}
|
||||
|
||||
/// Keeps the `limit` (at most [`MAX_LIMIT`]) first items of `hits` in
|
||||
/// `order`, sorted.
|
||||
fn best<T>(hits: &mut Vec<T>, limit: usize, order: impl Fn(&T, &T) -> Ordering) {
|
||||
let limit = limit.min(MAX_LIMIT);
|
||||
if limit == 0 {
|
||||
hits.clear();
|
||||
return;
|
||||
}
|
||||
if hits.len() > limit {
|
||||
hits.select_nth_unstable_by(limit.saturating_sub(1), &order);
|
||||
hits.truncate(limit);
|
||||
}
|
||||
hits.sort_by(order);
|
||||
}
|
||||
|
||||
/// The matcher configuration for symbol names: the default word
|
||||
/// boundaries (camel case, `\`, `_`, `:` and other punctuation start
|
||||
/// words), and matches near the start score a little higher.
|
||||
fn symbol_config() -> Config {
|
||||
let mut config = Config::DEFAULT;
|
||||
config.prefer_prefix = true;
|
||||
config
|
||||
}
|
||||
|
||||
/// The first [`MAX_QUERY_CHARS`] characters of `query`.
|
||||
fn clamp(query: &str) -> String {
|
||||
query.chars().take(MAX_QUERY_CHARS).collect()
|
||||
}
|
||||
|
||||
/// The file name of a project-relative path.
|
||||
fn file_name(path: &str) -> &str {
|
||||
path.rsplit('/').next().unwrap_or(path)
|
||||
}
|
||||
|
||||
/// Length of `text` in UTF-16 code units.
|
||||
fn utf16_len(text: &str) -> usize {
|
||||
text.chars().map(char::len_utf16).sum()
|
||||
}
|
||||
|
||||
/// Turns sorted character indices into `text` into merged `[start, end)`
|
||||
/// UTF-16 ranges, shifted by `base`.
|
||||
fn ranges(text: &str, indices: &[u32], base: usize) -> Vec<[usize; 2]> {
|
||||
let mut out: Vec<[usize; 2]> = Vec::new();
|
||||
let mut wanted = indices.iter().copied().peekable();
|
||||
let mut at = base;
|
||||
for (i, ch) in text.chars().enumerate() {
|
||||
let width = ch.len_utf16();
|
||||
if wanted
|
||||
.peek()
|
||||
.is_some_and(|&w| usize::try_from(w).is_ok_and(|w| w == i))
|
||||
{
|
||||
wanted.next();
|
||||
match out.last_mut() {
|
||||
Some(range) if range[1] == at => range[1] = at.saturating_add(width),
|
||||
_ => out.push([at, at.saturating_add(width)]),
|
||||
}
|
||||
}
|
||||
at = at.saturating_add(width);
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// A compiled query with the matcher and buffers it needs.
|
||||
struct Fuzzy {
|
||||
/// The query words.
|
||||
pattern: Pattern,
|
||||
/// The matcher and its scratch memory.
|
||||
matcher: Matcher,
|
||||
/// Characters of the current haystack, for non-ASCII text.
|
||||
chars: Vec<char>,
|
||||
/// Match indices, reused.
|
||||
indices: Vec<u32>,
|
||||
}
|
||||
|
||||
impl Fuzzy {
|
||||
/// The query `query` with `config`; `None` for a query without words.
|
||||
fn new(query: &str, config: Config) -> Option<Self> {
|
||||
let pattern = Pattern::new(
|
||||
query,
|
||||
CaseMatching::Smart,
|
||||
Normalization::Smart,
|
||||
AtomKind::Fuzzy,
|
||||
);
|
||||
if pattern.atoms.is_empty() {
|
||||
return None;
|
||||
}
|
||||
Some(Self {
|
||||
pattern,
|
||||
matcher: Matcher::new(config),
|
||||
chars: Vec::new(),
|
||||
indices: Vec::new(),
|
||||
})
|
||||
}
|
||||
|
||||
/// The score of `haystack`, `None` if it does not match.
|
||||
fn score(&mut self, haystack: &str) -> Option<u32> {
|
||||
let Self {
|
||||
pattern,
|
||||
matcher,
|
||||
chars,
|
||||
..
|
||||
} = self;
|
||||
pattern.score(utf32(haystack, chars), matcher)
|
||||
}
|
||||
|
||||
/// The matched character indices in `haystack`, sorted, without
|
||||
/// duplicates; empty if it does not match.
|
||||
fn indices(&mut self, haystack: &str) -> Vec<u32> {
|
||||
let Self {
|
||||
pattern,
|
||||
matcher,
|
||||
chars,
|
||||
indices,
|
||||
} = self;
|
||||
indices.clear();
|
||||
if pattern
|
||||
.indices(utf32(haystack, chars), matcher, indices)
|
||||
.is_none()
|
||||
{
|
||||
return Vec::new();
|
||||
}
|
||||
indices.sort_unstable();
|
||||
indices.dedup();
|
||||
indices.clone()
|
||||
}
|
||||
}
|
||||
|
||||
/// `text` as the matcher's string type, one entry per Unicode scalar
|
||||
/// value, so match indices are character indices (`Utf32Str::new` would
|
||||
/// merge grapheme clusters).
|
||||
fn utf32<'a>(text: &'a str, chars: &'a mut Vec<char>) -> Utf32Str<'a> {
|
||||
if text.is_ascii() {
|
||||
return Utf32Str::Ascii(text.as_bytes());
|
||||
}
|
||||
chars.clear();
|
||||
chars.extend(text.chars());
|
||||
Utf32Str::Unicode(chars)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::SymbolKind;
|
||||
|
||||
fn file(path: &str, symbols: &[(&str, &str, SymbolKind, Option<&str>)]) -> IndexedFile {
|
||||
IndexedFile {
|
||||
path: path.into(),
|
||||
library: crate::is_library(path),
|
||||
symbols: symbols
|
||||
.iter()
|
||||
.map(|(name, fqn, kind, container)| Symbol {
|
||||
name: (*name).into(),
|
||||
fqn: (*fqn).into(),
|
||||
kind: *kind,
|
||||
container: container.map(str::to_string),
|
||||
line: 1,
|
||||
column: 2,
|
||||
})
|
||||
.collect(),
|
||||
}
|
||||
}
|
||||
|
||||
fn sample() -> Index {
|
||||
let mut index = Index::new();
|
||||
index.insert(file(
|
||||
"src/Models/User.php",
|
||||
&[
|
||||
("User", "App\\Models\\User", SymbolKind::Class, None),
|
||||
(
|
||||
"find",
|
||||
"App\\Models\\User::find",
|
||||
SymbolKind::Method,
|
||||
Some("App\\Models\\User"),
|
||||
),
|
||||
(
|
||||
"$name",
|
||||
"App\\Models\\User::$name",
|
||||
SymbolKind::Property,
|
||||
Some("App\\Models\\User"),
|
||||
),
|
||||
],
|
||||
));
|
||||
index.insert(file(
|
||||
"src/user/Repository.php",
|
||||
&[(
|
||||
"Repository",
|
||||
"App\\User\\Repository",
|
||||
SymbolKind::Class,
|
||||
None,
|
||||
)],
|
||||
));
|
||||
index.insert(file(
|
||||
"vendor/acme/lib/src/User.php",
|
||||
&[
|
||||
("User", "Acme\\User", SymbolKind::Class, None),
|
||||
(
|
||||
"find",
|
||||
"Acme\\User::find",
|
||||
SymbolKind::Method,
|
||||
Some("Acme\\User"),
|
||||
),
|
||||
],
|
||||
));
|
||||
index.insert(file("templates/user.html.twig", &[]));
|
||||
index.insert(file(
|
||||
"src/helpers.php",
|
||||
&[("format_user", "format_user", SymbolKind::Function, None)],
|
||||
));
|
||||
index
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn counts_and_removal() {
|
||||
let mut index = sample();
|
||||
assert_eq!(index.len(), 5);
|
||||
assert_eq!(index.symbol_count(), 7);
|
||||
index.insert(file("src/helpers.php", &[]));
|
||||
assert_eq!(index.symbol_count(), 6);
|
||||
assert_eq!(index.remove("src/user"), 1);
|
||||
assert_eq!(index.remove("src/Models/User.php"), 1);
|
||||
assert_eq!(index.remove("src/Models"), 0);
|
||||
assert_eq!(index.symbol_count(), 2);
|
||||
assert!(index.contains("src/helpers.php"));
|
||||
assert_eq!(index.remove(""), 3);
|
||||
assert!(index.is_empty());
|
||||
assert_eq!(index.symbol_count(), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prefix_removal_respects_segments() {
|
||||
let mut index = Index::new();
|
||||
index.insert(file("src/a.php", &[]));
|
||||
index.insert(file("src2/b.php", &[]));
|
||||
index.insert(file("src-x/c.php", &[]));
|
||||
assert_eq!(index.remove("src"), 1);
|
||||
assert_eq!(index.len(), 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn files_rank_names_then_paths_and_project_first() {
|
||||
let index = sample();
|
||||
let paths: Vec<String> = index
|
||||
.find_files("user", 10)
|
||||
.into_iter()
|
||||
.map(|h| h.path)
|
||||
.collect();
|
||||
assert_eq!(
|
||||
paths,
|
||||
vec![
|
||||
"src/Models/User.php",
|
||||
"templates/user.html.twig",
|
||||
"src/user/Repository.php",
|
||||
"vendor/acme/lib/src/User.php",
|
||||
]
|
||||
);
|
||||
let hit = &index.find_files("user", 1)[0];
|
||||
assert_eq!(hit.matches, vec![[11, 15]]);
|
||||
let by_path = index.find_files("models/u", 10);
|
||||
assert_eq!(by_path[0].path, "src/Models/User.php");
|
||||
assert!(index.find_files(" ", 10).is_empty());
|
||||
assert!(index.find_files("user", 0).is_empty());
|
||||
assert!(index.find_files("zzz", 10).is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn classes_and_symbols() {
|
||||
let index = sample();
|
||||
let classes: Vec<(&str, bool)> = index
|
||||
.find_classes("user", 10)
|
||||
.iter()
|
||||
.map(|h| (h.symbol.fqn.as_str(), h.library))
|
||||
.collect();
|
||||
assert_eq!(
|
||||
classes,
|
||||
vec![("App\\Models\\User", false), ("Acme\\User", true)]
|
||||
);
|
||||
let fqn = index.find_classes("\\acme\\us", 10);
|
||||
assert_eq!(fqn.len(), 1);
|
||||
assert_eq!(fqn[0].symbol.fqn, "Acme\\User");
|
||||
let finds: Vec<&str> = index
|
||||
.find_symbols("find", 10)
|
||||
.iter()
|
||||
.map(|h| h.symbol.fqn.as_str())
|
||||
.collect();
|
||||
assert_eq!(finds, vec!["App\\Models\\User::find", "Acme\\User::find"]);
|
||||
let members: Vec<&str> = index
|
||||
.find_symbols("User::", 10)
|
||||
.iter()
|
||||
.map(|h| h.symbol.name.as_str())
|
||||
.collect();
|
||||
assert_eq!(members, vec!["find", "$name", "find"]);
|
||||
let scoped = index.find_symbols("acme\\user::f", 10);
|
||||
assert_eq!(scoped.len(), 1);
|
||||
assert!(
|
||||
index
|
||||
.find_symbols("user", 10)
|
||||
.iter()
|
||||
.all(|h| !h.symbol.kind.is_class_like())
|
||||
);
|
||||
assert!(index.find_symbols("::", 10).is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn match_ranges_count_utf16() {
|
||||
assert_eq!(ranges("aéb", &[0, 1, 2], 0), vec![[0, 3]]);
|
||||
assert_eq!(ranges("a😀b", &[2], 4), vec![[7, 8]]);
|
||||
assert_eq!(ranges("abc", &[0, 2], 0), vec![[0, 1], [2, 3]]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn caps_the_number_of_files() {
|
||||
let mut index = Index::new();
|
||||
for i in 0..MAX_FILES {
|
||||
index.files.insert(format!("f{i}"), IndexedFile::default());
|
||||
}
|
||||
assert!(!index.insert(file("new.php", &[])));
|
||||
assert!(index.insert(file("f1", &[])));
|
||||
}
|
||||
}
|
||||
313
crates/index/src/symbol.rs
Normal file
313
crates/index/src/symbol.rs
Normal file
|
|
@ -0,0 +1,313 @@
|
|||
//! Symbols and indexed files, extracted from tree-sitter outlines.
|
||||
|
||||
use lutra_syntax::{OutlineItem, OutlineKind, Parser};
|
||||
|
||||
use crate::{MAX_PARSE_BYTES, MAX_SYMBOLS_PER_FILE};
|
||||
|
||||
/// What a symbol declares.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
|
||||
pub enum SymbolKind {
|
||||
/// A class.
|
||||
Class,
|
||||
/// An interface.
|
||||
Interface,
|
||||
/// A trait.
|
||||
Trait,
|
||||
/// An enum.
|
||||
Enum,
|
||||
/// A function outside a class.
|
||||
Function,
|
||||
/// A method.
|
||||
Method,
|
||||
/// A property, declared or constructor-promoted.
|
||||
Property,
|
||||
/// A class constant or a `const` outside a class.
|
||||
Constant,
|
||||
/// A `case` of an enum.
|
||||
EnumCase,
|
||||
}
|
||||
|
||||
impl SymbolKind {
|
||||
/// The kind's name as the engine and the UI use it, the same as the
|
||||
/// outline's: `class`, `interface`, `trait`, `enum`, `function`,
|
||||
/// `method`, `property`, `constant` or `enumCase`.
|
||||
pub fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::Class => "class",
|
||||
Self::Interface => "interface",
|
||||
Self::Trait => "trait",
|
||||
Self::Enum => "enum",
|
||||
Self::Function => "function",
|
||||
Self::Method => "method",
|
||||
Self::Property => "property",
|
||||
Self::Constant => "constant",
|
||||
Self::EnumCase => "enumCase",
|
||||
}
|
||||
}
|
||||
|
||||
/// Whether the kind is a class, interface, trait or enum (Go to Class);
|
||||
/// the others are found by Go to Symbol.
|
||||
pub fn is_class_like(self) -> bool {
|
||||
matches!(
|
||||
self,
|
||||
Self::Class | Self::Interface | Self::Trait | Self::Enum
|
||||
)
|
||||
}
|
||||
|
||||
/// The symbol kind of an outline kind; `None` for a namespace.
|
||||
fn from_outline(kind: OutlineKind) -> Option<Self> {
|
||||
Some(match kind {
|
||||
OutlineKind::Namespace => return None,
|
||||
OutlineKind::Class => Self::Class,
|
||||
OutlineKind::Interface => Self::Interface,
|
||||
OutlineKind::Trait => Self::Trait,
|
||||
OutlineKind::Enum => Self::Enum,
|
||||
OutlineKind::Function => Self::Function,
|
||||
OutlineKind::Method => Self::Method,
|
||||
OutlineKind::Property => Self::Property,
|
||||
OutlineKind::Constant => Self::Constant,
|
||||
OutlineKind::EnumCase => Self::EnumCase,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// One declaration.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
|
||||
pub struct Symbol {
|
||||
/// The name as written: `Product`, `find`, `$id` (properties keep the
|
||||
/// `$`), `MAX`.
|
||||
pub name: String,
|
||||
/// The fully qualified name without a leading `\`: `App\Models\Product`
|
||||
/// for a class or function, `App\Models\Product::find` for a member.
|
||||
pub fqn: String,
|
||||
/// What it declares.
|
||||
pub kind: SymbolKind,
|
||||
/// The FQN of the class, interface, trait or enum that declares a
|
||||
/// member; `None` for classes, functions and constants outside a class.
|
||||
pub container: Option<String>,
|
||||
/// 0-based line of the name.
|
||||
pub line: usize,
|
||||
/// Column of the name in UTF-16 code units.
|
||||
pub column: usize,
|
||||
}
|
||||
|
||||
/// A file in the index.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Default)]
|
||||
pub struct IndexedFile {
|
||||
/// Project-relative path with `/` separators.
|
||||
pub path: String,
|
||||
/// Library code: the file is below the root's `vendor/`.
|
||||
pub library: bool,
|
||||
/// Declarations in source order; empty for files that are not parsed.
|
||||
pub symbols: Vec<Symbol>,
|
||||
}
|
||||
|
||||
/// Whether the project-relative `path` is library code (the root's
|
||||
/// `vendor/` or anything below it).
|
||||
pub fn is_library(path: &str) -> bool {
|
||||
path == "vendor" || path.starts_with("vendor/")
|
||||
}
|
||||
|
||||
/// Whether the project-relative `path` is a PHP file to parse.
|
||||
pub(crate) fn is_php(path: &str) -> bool {
|
||||
path.rsplit('/')
|
||||
.next()
|
||||
.and_then(|name| name.rsplit_once('.'))
|
||||
.is_some_and(|(stem, ext)| !stem.is_empty() && ext.eq_ignore_ascii_case("php"))
|
||||
}
|
||||
|
||||
/// Indexes `text` as the file at `path`: parses it when it is a PHP file
|
||||
/// of at most [`MAX_PARSE_BYTES`] and extracts its symbols; other files get
|
||||
/// no symbols.
|
||||
///
|
||||
/// # Parameters
|
||||
/// - `parser`: reused between files.
|
||||
/// - `path`: project-relative path with `/` separators.
|
||||
/// - `text`: the file's text.
|
||||
pub fn index_text(parser: &mut Parser, path: String, text: &str) -> IndexedFile {
|
||||
let library = is_library(&path);
|
||||
let parse = is_php(&path) && text.len() as u64 <= MAX_PARSE_BYTES;
|
||||
let symbols = if parse {
|
||||
let tree = parser.parse(text);
|
||||
symbols_from_outline(&lutra_syntax::outline(&tree))
|
||||
} else {
|
||||
Vec::new()
|
||||
};
|
||||
IndexedFile {
|
||||
path,
|
||||
library,
|
||||
symbols,
|
||||
}
|
||||
}
|
||||
|
||||
/// The symbols declared in an outline, in source order, at most
|
||||
/// [`MAX_SYMBOLS_PER_FILE`].
|
||||
///
|
||||
/// Namespaces give the FQN prefix of what they contain; classes,
|
||||
/// interfaces, traits and enums are the container of their members.
|
||||
/// Anonymous classes and their members are skipped: nothing can navigate to
|
||||
/// them by name.
|
||||
pub fn symbols_from_outline(items: &[OutlineItem]) -> Vec<Symbol> {
|
||||
let mut out = Vec::new();
|
||||
collect(items, "", None, &mut out);
|
||||
out
|
||||
}
|
||||
|
||||
/// Adds the symbols of `items` to `out`.
|
||||
///
|
||||
/// # Parameters
|
||||
/// - `namespace`: the enclosing namespace without leading or trailing `\`;
|
||||
/// empty for the global namespace.
|
||||
/// - `container`: the FQN of the enclosing class-like declaration.
|
||||
fn collect(items: &[OutlineItem], namespace: &str, container: Option<&str>, out: &mut Vec<Symbol>) {
|
||||
for item in items {
|
||||
if out.len() >= MAX_SYMBOLS_PER_FILE {
|
||||
return;
|
||||
}
|
||||
let Some(kind) = SymbolKind::from_outline(item.kind) else {
|
||||
let name = item.name.trim_matches('\\');
|
||||
collect(&item.children, name, None, out);
|
||||
continue;
|
||||
};
|
||||
if item.is_anonymous() || item.name.is_empty() {
|
||||
continue;
|
||||
}
|
||||
let (fqn, owner) = match container {
|
||||
Some(owner) if !kind.is_class_like() && kind != SymbolKind::Function => {
|
||||
(format!("{owner}::{}", item.name), Some(owner.to_string()))
|
||||
}
|
||||
_ => (qualify(namespace, &item.name), None),
|
||||
};
|
||||
let start = item.selection_range.start;
|
||||
out.push(Symbol {
|
||||
name: item.name.clone(),
|
||||
fqn: fqn.clone(),
|
||||
kind,
|
||||
container: owner,
|
||||
line: start.line,
|
||||
column: start.column,
|
||||
});
|
||||
if kind.is_class_like() {
|
||||
collect(&item.children, namespace, Some(&fqn), out);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// `name` qualified with `namespace`: `App\Product`, or `Product` in the
|
||||
/// global namespace.
|
||||
fn qualify(namespace: &str, name: &str) -> String {
|
||||
let name = name.trim_start_matches('\\');
|
||||
if namespace.is_empty() {
|
||||
name.to_string()
|
||||
} else {
|
||||
format!("{namespace}\\{name}")
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn symbols(source: &str) -> Vec<Symbol> {
|
||||
let mut parser = Parser::new().unwrap();
|
||||
index_text(&mut parser, "src/a.php".into(), source).symbols
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn qualifies_classes_members_and_functions() {
|
||||
let found = symbols(
|
||||
"<?php\nnamespace App\\Models;\n\ninterface HasId {}\nfinal class Product implements HasId\n{\n public const MAX = 3;\n private ?string $name = null;\n public function __construct(private int $id) {}\n public function find(int $id): ?self { return null; }\n}\nenum Status: string { case Draft = 'draft'; }\nfunction helper(): void {}\nconst VERSION = 1;\n$x = new class { public function hidden() {} };\n",
|
||||
);
|
||||
let summary: Vec<(String, &str, Option<String>)> = found
|
||||
.iter()
|
||||
.map(|s| (s.fqn.clone(), s.kind.as_str(), s.container.clone()))
|
||||
.collect();
|
||||
let product = Some("App\\Models\\Product".to_string());
|
||||
assert_eq!(
|
||||
summary,
|
||||
vec![
|
||||
("App\\Models\\HasId".into(), "interface", None),
|
||||
("App\\Models\\Product".into(), "class", None),
|
||||
(
|
||||
"App\\Models\\Product::MAX".into(),
|
||||
"constant",
|
||||
product.clone()
|
||||
),
|
||||
(
|
||||
"App\\Models\\Product::$name".into(),
|
||||
"property",
|
||||
product.clone()
|
||||
),
|
||||
(
|
||||
"App\\Models\\Product::__construct".into(),
|
||||
"method",
|
||||
product.clone()
|
||||
),
|
||||
(
|
||||
"App\\Models\\Product::$id".into(),
|
||||
"property",
|
||||
product.clone()
|
||||
),
|
||||
("App\\Models\\Product::find".into(), "method", product),
|
||||
("App\\Models\\Status".into(), "enum", None),
|
||||
(
|
||||
"App\\Models\\Status::Draft".into(),
|
||||
"enumCase",
|
||||
Some("App\\Models\\Status".into())
|
||||
),
|
||||
("App\\Models\\helper".into(), "function", None),
|
||||
("App\\Models\\VERSION".into(), "constant", None),
|
||||
]
|
||||
);
|
||||
let product = found.iter().find(|s| s.name == "Product").unwrap();
|
||||
assert_eq!((product.line, product.column), (4, 12));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn braced_and_global_namespaces() {
|
||||
let found = symbols(
|
||||
"<?php\nnamespace A { class One {} }\nnamespace { class Two {} function three() {} }\n",
|
||||
);
|
||||
let fqns: Vec<&str> = found.iter().map(|s| s.fqn.as_str()).collect();
|
||||
assert_eq!(fqns, vec!["A\\One", "Two", "three"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn only_php_files_are_parsed() {
|
||||
let mut parser = Parser::new().unwrap();
|
||||
let text = "<?php class A {}";
|
||||
assert_eq!(
|
||||
index_text(&mut parser, "a.php".into(), text).symbols.len(),
|
||||
1
|
||||
);
|
||||
assert_eq!(
|
||||
index_text(&mut parser, "A.PHP".into(), text).symbols.len(),
|
||||
1
|
||||
);
|
||||
assert!(
|
||||
index_text(&mut parser, "a.twig".into(), text)
|
||||
.symbols
|
||||
.is_empty()
|
||||
);
|
||||
assert!(
|
||||
index_text(&mut parser, ".php".into(), text)
|
||||
.symbols
|
||||
.is_empty()
|
||||
);
|
||||
let file = index_text(&mut parser, "vendor/x/a.php".into(), text);
|
||||
assert!(file.library);
|
||||
assert!(!is_library("vendors/a.php"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn oversized_files_have_no_symbols() {
|
||||
let mut parser = Parser::new().unwrap();
|
||||
let mut text = String::from("<?php class A {}\n");
|
||||
text.push_str(&"// filler\n".repeat((MAX_PARSE_BYTES as usize) / 10 + 1));
|
||||
assert!(
|
||||
index_text(&mut parser, "a.php".into(), &text)
|
||||
.symbols
|
||||
.is_empty()
|
||||
);
|
||||
}
|
||||
}
|
||||
192
crates/index/src/updater.rs
Normal file
192
crates/index/src/updater.rs
Normal file
|
|
@ -0,0 +1,192 @@
|
|||
//! Incremental updates: re-indexing single files and directories after the
|
||||
//! build, on a background thread.
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::path::PathBuf;
|
||||
use std::sync::mpsc::{Receiver, Sender, channel};
|
||||
|
||||
use lutra_syntax::Parser;
|
||||
use lutra_text::Buffer;
|
||||
|
||||
use crate::build::{guarded_index, index_file};
|
||||
use crate::walk::walk;
|
||||
use crate::{CancelToken, IndexedFile, MAX_FILES, Update};
|
||||
|
||||
/// Most files one directory rescan reports in one [`Update::Files`].
|
||||
const RESCAN_BATCH: usize = 256;
|
||||
|
||||
/// Work for an [`Updater`].
|
||||
#[derive(Debug, Clone)]
|
||||
pub enum Job {
|
||||
/// Re-index one file.
|
||||
File {
|
||||
/// Project-relative path with `/` separators.
|
||||
path: String,
|
||||
/// The text of an open editor buffer to index instead of the file on
|
||||
/// disk; `None` reads the disk, and reports [`Update::Removed`] when
|
||||
/// the file is gone or now excluded by the project's rules.
|
||||
text: Option<Buffer>,
|
||||
},
|
||||
/// Re-index a directory: drop everything below it, then list it again
|
||||
/// from disk.
|
||||
Dir {
|
||||
/// Project-relative path with `/` separators.
|
||||
path: String,
|
||||
},
|
||||
}
|
||||
|
||||
impl Job {
|
||||
/// The path the job is about.
|
||||
fn path(&self) -> &str {
|
||||
match self {
|
||||
Self::File { path, .. } | Self::Dir { path } => path,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// A background thread that runs [`Job`]s one batch at a time and sends
|
||||
/// the results as live [`Update`]s. Jobs queued while it works are
|
||||
/// coalesced: only the last job per path runs, so a burst of keystrokes in
|
||||
/// one buffer parses it once. The thread ends when the updater is dropped
|
||||
/// or `cancel` fires.
|
||||
#[derive(Debug)]
|
||||
pub struct Updater {
|
||||
/// Queues jobs for the thread.
|
||||
jobs: Sender<Job>,
|
||||
}
|
||||
|
||||
impl Updater {
|
||||
/// Starts the thread for the project at `root`.
|
||||
///
|
||||
/// # Parameters
|
||||
/// - `root`: the project root (absolute).
|
||||
/// - `cancel`: stops the thread between jobs.
|
||||
/// - `updates`: where results go, usually the build's channel.
|
||||
///
|
||||
/// # Errors
|
||||
/// The thread could not be started.
|
||||
pub fn spawn(
|
||||
root: PathBuf,
|
||||
cancel: CancelToken,
|
||||
updates: Sender<Update>,
|
||||
) -> std::io::Result<Self> {
|
||||
let (jobs, queue) = channel();
|
||||
std::thread::Builder::new()
|
||||
.name("lutra-index-update".into())
|
||||
.spawn(move || run(&root, &cancel, &queue, &updates))?;
|
||||
Ok(Self { jobs })
|
||||
}
|
||||
|
||||
/// Queues `job`. Returns `false` if the thread has ended.
|
||||
pub fn send(&self, job: Job) -> bool {
|
||||
self.jobs.send(job).is_ok()
|
||||
}
|
||||
}
|
||||
|
||||
/// The updater thread's loop.
|
||||
fn run(
|
||||
root: &std::path::Path,
|
||||
cancel: &CancelToken,
|
||||
queue: &Receiver<Job>,
|
||||
updates: &Sender<Update>,
|
||||
) {
|
||||
let mut parser = Parser::new().ok();
|
||||
while let Ok(first) = queue.recv() {
|
||||
let mut jobs = vec![first];
|
||||
jobs.extend(queue.try_iter());
|
||||
for job in coalesce(jobs) {
|
||||
if cancel.is_cancelled() {
|
||||
return;
|
||||
}
|
||||
let sent = match job {
|
||||
Job::File {
|
||||
path,
|
||||
text: Some(text),
|
||||
} => {
|
||||
let library = crate::is_library(&path);
|
||||
let file = guarded_index(&mut parser, path.clone(), &text.text()).unwrap_or(
|
||||
IndexedFile {
|
||||
path,
|
||||
library,
|
||||
symbols: Vec::new(),
|
||||
},
|
||||
);
|
||||
updates
|
||||
.send(Update::Files {
|
||||
files: vec![file],
|
||||
live: true,
|
||||
})
|
||||
.is_ok()
|
||||
}
|
||||
Job::File { path, text: None } => {
|
||||
rescan(root, &path, false, cancel, &mut parser, updates)
|
||||
}
|
||||
Job::Dir { path } => rescan(root, &path, true, cancel, &mut parser, updates),
|
||||
};
|
||||
if !sent {
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Keeps the last job per path, in the order of those last jobs.
|
||||
fn coalesce(jobs: Vec<Job>) -> Vec<Job> {
|
||||
let mut last: HashMap<String, usize> = HashMap::new();
|
||||
for (i, job) in jobs.iter().enumerate() {
|
||||
last.insert(job.path().to_string(), i);
|
||||
}
|
||||
jobs.into_iter()
|
||||
.enumerate()
|
||||
.filter(|(i, job)| last.get(job.path()) == Some(i))
|
||||
.map(|(_, job)| job)
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Lists `path` from disk with the project's rules and indexes what is
|
||||
/// found. A file that is no longer found is reported removed; a directory
|
||||
/// is always reported removed first, so files gone from it disappear.
|
||||
/// Returns `false` once the receiver is gone.
|
||||
fn rescan(
|
||||
root: &std::path::Path,
|
||||
path: &str,
|
||||
dir: bool,
|
||||
cancel: &CancelToken,
|
||||
parser: &mut Option<Parser>,
|
||||
updates: &Sender<Update>,
|
||||
) -> bool {
|
||||
let walked = walk(root, Some(path), 1, MAX_FILES, cancel);
|
||||
if (dir || walked.files.is_empty())
|
||||
&& updates
|
||||
.send(Update::Removed {
|
||||
path: path.to_string(),
|
||||
})
|
||||
.is_err()
|
||||
{
|
||||
return false;
|
||||
}
|
||||
let mut batch = Vec::new();
|
||||
for found in &walked.files {
|
||||
if cancel.is_cancelled() {
|
||||
return true;
|
||||
}
|
||||
batch.push(index_file(parser, found).0);
|
||||
if batch.len() >= RESCAN_BATCH
|
||||
&& updates
|
||||
.send(Update::Files {
|
||||
files: std::mem::take(&mut batch),
|
||||
live: true,
|
||||
})
|
||||
.is_err()
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
batch.is_empty()
|
||||
|| updates
|
||||
.send(Update::Files {
|
||||
files: batch,
|
||||
live: true,
|
||||
})
|
||||
.is_ok()
|
||||
}
|
||||
193
crates/index/src/walk.rs
Normal file
193
crates/index/src/walk.rs
Normal file
|
|
@ -0,0 +1,193 @@
|
|||
//! The walk over the project: which files the index lists (see the crate
|
||||
//! docs for the rules).
|
||||
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::Mutex;
|
||||
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
|
||||
|
||||
use ignore::{DirEntry, WalkBuilder, WalkState};
|
||||
|
||||
use crate::CancelToken;
|
||||
use crate::symbol::is_php;
|
||||
|
||||
/// A file the walk found.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub(crate) struct Found {
|
||||
/// Project-relative path with `/` separators.
|
||||
pub path: String,
|
||||
/// Absolute path.
|
||||
pub abs: PathBuf,
|
||||
/// Size in bytes for PHP files; 0 for other files, which are not read.
|
||||
pub size: u64,
|
||||
}
|
||||
|
||||
/// What a walk returns.
|
||||
#[derive(Debug, Default)]
|
||||
pub(crate) struct Walked {
|
||||
/// The files, in no particular order.
|
||||
pub files: Vec<Found>,
|
||||
/// More than the limit were found; the rest are left out.
|
||||
pub capped: bool,
|
||||
}
|
||||
|
||||
/// Lists the files below `root`, or only those at or below the
|
||||
/// project-relative `scope` (a file or a directory), with the project's
|
||||
/// rules.
|
||||
///
|
||||
/// # Parameters
|
||||
/// - `root`: the project root (absolute).
|
||||
/// - `scope`: `None` for the whole project.
|
||||
/// - `threads`: walker threads.
|
||||
/// - `limit`: most files to return.
|
||||
/// - `cancel`: stops the walk.
|
||||
///
|
||||
/// Entries that cannot be read are skipped, as the project tree skips
|
||||
/// them.
|
||||
pub(crate) fn walk(
|
||||
root: &Path,
|
||||
scope: Option<&str>,
|
||||
threads: usize,
|
||||
limit: usize,
|
||||
cancel: &CancelToken,
|
||||
) -> Walked {
|
||||
let found = Mutex::new(Vec::new());
|
||||
let count = AtomicUsize::new(0);
|
||||
let capped = AtomicBool::new(false);
|
||||
let scope = scope.map(|s| s.trim_matches('/').to_string());
|
||||
walker(root, scope.clone(), threads)
|
||||
.build_parallel()
|
||||
.run(|| {
|
||||
let found = &found;
|
||||
let count = &count;
|
||||
let capped = &capped;
|
||||
let scope = scope.as_deref();
|
||||
Box::new(move |entry| {
|
||||
if cancel.is_cancelled() || capped.load(Ordering::Relaxed) {
|
||||
return WalkState::Quit;
|
||||
}
|
||||
let Ok(entry) = entry else {
|
||||
return WalkState::Continue;
|
||||
};
|
||||
if !entry.file_type().is_some_and(|t| t.is_file()) {
|
||||
return WalkState::Continue;
|
||||
}
|
||||
let Some(rel) = relative(root, entry.path()) else {
|
||||
return WalkState::Continue;
|
||||
};
|
||||
if scope.is_some_and(|s| !inside(&rel, s)) {
|
||||
return WalkState::Continue;
|
||||
}
|
||||
if count.fetch_add(1, Ordering::Relaxed) >= limit {
|
||||
capped.store(true, Ordering::Relaxed);
|
||||
return WalkState::Quit;
|
||||
}
|
||||
let size = if is_php(&rel) {
|
||||
entry.metadata().map(|m| m.len()).unwrap_or(0)
|
||||
} else {
|
||||
0
|
||||
};
|
||||
if let Ok(mut files) = found.lock() {
|
||||
files.push(Found {
|
||||
path: rel,
|
||||
abs: entry.path().to_path_buf(),
|
||||
size,
|
||||
});
|
||||
}
|
||||
WalkState::Continue
|
||||
})
|
||||
});
|
||||
Walked {
|
||||
files: found.into_inner().unwrap_or_default(),
|
||||
capped: capped.load(Ordering::Relaxed),
|
||||
}
|
||||
}
|
||||
|
||||
/// The walker over `root` with the project's rules, entering only
|
||||
/// directories on the way to or inside `scope`.
|
||||
fn walker(root: &Path, scope: Option<String>, threads: usize) -> WalkBuilder {
|
||||
let mut builder = WalkBuilder::new(root);
|
||||
builder
|
||||
.hidden(false)
|
||||
.parents(false)
|
||||
.ignore(true)
|
||||
.git_ignore(true)
|
||||
.git_exclude(true)
|
||||
.git_global(true)
|
||||
.require_git(false)
|
||||
.follow_links(false)
|
||||
.threads(threads.max(1));
|
||||
let vendor = root.join("vendor");
|
||||
let vendor_is_dir = std::fs::symlink_metadata(&vendor).is_ok_and(|m| m.is_dir());
|
||||
let vendor_in_scope = scope
|
||||
.as_deref()
|
||||
.is_none_or(|s| related("vendor", s) || s.is_empty());
|
||||
if vendor_is_dir && vendor_in_scope {
|
||||
builder.add(&vendor);
|
||||
}
|
||||
let root = root.to_path_buf();
|
||||
builder.filter_entry(move |entry| keep_entry(&root, &vendor, scope.as_deref(), entry));
|
||||
builder
|
||||
}
|
||||
|
||||
/// Whether the walk enters (directory) or visits (file) `entry`: `.git`
|
||||
/// and `node_modules` are skipped, the root's `vendor` is reached through
|
||||
/// its own walk root, and with a scope only directories on the way to or
|
||||
/// inside it are entered.
|
||||
fn keep_entry(root: &Path, vendor: &Path, scope: Option<&str>, entry: &DirEntry) -> bool {
|
||||
let name = entry.file_name();
|
||||
if name == ".git" {
|
||||
return false;
|
||||
}
|
||||
let is_dir = entry.file_type().is_some_and(|t| t.is_dir());
|
||||
if is_dir && name == "node_modules" {
|
||||
return false;
|
||||
}
|
||||
if entry.depth() == 1 && entry.path() == vendor {
|
||||
return false;
|
||||
}
|
||||
if is_dir
|
||||
&& let Some(scope) = scope
|
||||
&& let Some(rel) = relative(root, entry.path())
|
||||
{
|
||||
return related(&rel, scope);
|
||||
}
|
||||
true
|
||||
}
|
||||
|
||||
/// Whether `path` is `scope` or below it.
|
||||
fn inside(path: &str, scope: &str) -> bool {
|
||||
scope.is_empty()
|
||||
|| path == scope
|
||||
|| path
|
||||
.strip_prefix(scope)
|
||||
.is_some_and(|rest| rest.starts_with('/'))
|
||||
}
|
||||
|
||||
/// Whether the directory `dir` is on the way to `scope` or inside it.
|
||||
fn related(dir: &str, scope: &str) -> bool {
|
||||
inside(dir, scope) || inside(scope, dir)
|
||||
}
|
||||
|
||||
/// `path` relative to `root` with `/` separators; `None` outside the root
|
||||
/// or for a name that is not valid Unicode.
|
||||
fn relative(root: &Path, path: &Path) -> Option<String> {
|
||||
let rel = path.strip_prefix(root).ok()?;
|
||||
let parts: Option<Vec<&str>> = rel.components().map(|c| c.as_os_str().to_str()).collect();
|
||||
Some(parts?.join("/"))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn scope_relations() {
|
||||
assert!(inside("src/a.php", "src"));
|
||||
assert!(inside("src", "src"));
|
||||
assert!(!inside("src2/a.php", "src"));
|
||||
assert!(inside("anything", ""));
|
||||
assert!(related("src", "src/deep/a.php"));
|
||||
assert!(related("src/deep/x", "src/deep"));
|
||||
assert!(!related("lib", "src/a.php"));
|
||||
}
|
||||
}
|
||||
222
crates/index/tests/index.rs
Normal file
222
crates/index/tests/index.rs
Normal file
|
|
@ -0,0 +1,222 @@
|
|||
//! Building the index of a project on disk, incremental updates, and
|
||||
//! cancellation.
|
||||
|
||||
#![allow(
|
||||
clippy::unwrap_used,
|
||||
clippy::expect_used,
|
||||
clippy::indexing_slicing,
|
||||
clippy::panic
|
||||
)]
|
||||
|
||||
use std::fs;
|
||||
use std::path::Path;
|
||||
use std::sync::Mutex;
|
||||
use std::sync::mpsc::{Receiver, channel};
|
||||
use std::time::Duration;
|
||||
|
||||
use lutra_index::{CancelToken, Index, Job, Update, Updater, build, spawn_build};
|
||||
|
||||
/// Builds the index of `root` and applies every update.
|
||||
fn build_index(root: &Path, threads: usize) -> (Index, lutra_index::Summary, Vec<(usize, usize)>) {
|
||||
let updates = Mutex::new(Vec::new());
|
||||
let summary = build(root, threads, &CancelToken::new(), &|u| {
|
||||
updates.lock().unwrap().push(u)
|
||||
});
|
||||
let mut index = Index::new();
|
||||
let mut progress = Vec::new();
|
||||
for update in updates.into_inner().unwrap() {
|
||||
match update {
|
||||
Update::Files { files, live } => {
|
||||
assert!(!live);
|
||||
for file in files {
|
||||
assert!(index.insert(file));
|
||||
}
|
||||
}
|
||||
Update::Progress { done, total } => progress.push((done, total)),
|
||||
other => panic!("unexpected {other:?}"),
|
||||
}
|
||||
}
|
||||
(index, summary, progress)
|
||||
}
|
||||
|
||||
/// Applies updates from `rx` until one batch arrives and `rx` is quiet.
|
||||
fn apply_until_quiet(index: &mut Index, rx: &Receiver<Update>) {
|
||||
let first = rx.recv_timeout(Duration::from_secs(10)).expect("an update");
|
||||
let mut pending = vec![first];
|
||||
while let Ok(next) = rx.recv_timeout(Duration::from_millis(300)) {
|
||||
pending.push(next);
|
||||
}
|
||||
for update in pending {
|
||||
match update {
|
||||
Update::Files { files, live } => {
|
||||
assert!(live);
|
||||
for file in files {
|
||||
index.insert(file);
|
||||
}
|
||||
}
|
||||
Update::Removed { path } => {
|
||||
index.remove(&path);
|
||||
}
|
||||
other => panic!("unexpected {other:?}"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn write(root: &Path, rel: &str, text: &str) {
|
||||
let path = root.join(rel);
|
||||
fs::create_dir_all(path.parent().unwrap()).unwrap();
|
||||
fs::write(path, text).unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn builds_with_the_project_rules() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let project = lutra_testkit::generate_php_project(dir.path(), 400);
|
||||
write(
|
||||
dir.path(),
|
||||
"src/Broken.php",
|
||||
"<?php\nclass Broken {\n public function ok() {}\n public function bad() { $x = ; }\n",
|
||||
);
|
||||
write(dir.path(), "src/Latin1.php", "<?php\nclass Caf\u{e9} {}\n");
|
||||
fs::write(
|
||||
dir.path().join("src/Bad.php"),
|
||||
b"<?php\nclass Bad\xff {}\nclass Fine {}\n",
|
||||
)
|
||||
.unwrap();
|
||||
write(dir.path(), ".git/Hidden.php", "<?php class InGit {}");
|
||||
for threads in [1, 4] {
|
||||
let (index, summary, progress) = build_index(dir.path(), threads);
|
||||
assert_eq!(index.len(), project.indexed_files + 3);
|
||||
assert_eq!(summary.files, index.len());
|
||||
assert_eq!(summary.symbols, index.symbol_count());
|
||||
assert_eq!(summary.parsed, project.php_files + 3);
|
||||
assert!(!summary.capped && !summary.cancelled);
|
||||
let (done, total) = *progress.last().unwrap();
|
||||
assert_eq!((done, total), (index.len(), index.len()));
|
||||
assert!(progress.windows(2).all(|w| w[0].0 <= w[1].0));
|
||||
assert!(index.get("var/cache/Cached0.php").is_none(), "gitignored");
|
||||
assert!(
|
||||
index.get("node_modules/pkg/Hidden0.php").is_none(),
|
||||
"excluded"
|
||||
);
|
||||
assert!(index.get(".git/Hidden.php").is_none());
|
||||
let vendor = index
|
||||
.get("vendor/acme/framework/src/Component000/Widget0.php")
|
||||
.unwrap();
|
||||
assert!(vendor.library, "vendor is walked although gitignored");
|
||||
let classes = index.find_classes("Service1", 200);
|
||||
let service = classes
|
||||
.iter()
|
||||
.find(|h| h.symbol.fqn == "App\\Module000\\Service1")
|
||||
.unwrap();
|
||||
assert_eq!(service.path, "src/Module000/Service1.php");
|
||||
assert_eq!((service.symbol.line, service.symbol.column), (12, 6));
|
||||
assert_eq!(index.find_classes("Broken", 5)[0].symbol.name, "Broken");
|
||||
assert!(!index.find_symbols("Broken::ok", 5).is_empty());
|
||||
assert_eq!(index.find_classes("Café", 5)[0].symbol.name, "Café");
|
||||
assert!(!index.find_classes("Fine", 5).is_empty());
|
||||
let classes_found = index.find_classes("", 10);
|
||||
assert!(classes_found.is_empty());
|
||||
let all: usize = index
|
||||
.find_classes("e", 1_000)
|
||||
.iter()
|
||||
.filter(|h| h.symbol.kind.is_class_like())
|
||||
.count();
|
||||
assert!(all > 0);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn spawned_build_ends_with_ready_and_cancels() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let project = lutra_testkit::generate_php_project(dir.path(), 300);
|
||||
let (tx, rx) = channel();
|
||||
spawn_build(dir.path().to_path_buf(), 2, CancelToken::new(), tx).unwrap();
|
||||
let mut files = 0;
|
||||
let summary = loop {
|
||||
match rx.recv_timeout(Duration::from_secs(30)).unwrap() {
|
||||
Update::Files { files: batch, .. } => files += batch.len(),
|
||||
Update::Ready(summary) => break summary,
|
||||
_ => {}
|
||||
}
|
||||
};
|
||||
assert_eq!(files, project.indexed_files);
|
||||
assert_eq!(summary.files, files);
|
||||
assert!(rx.recv_timeout(Duration::from_millis(200)).is_err());
|
||||
|
||||
let cancel = CancelToken::new();
|
||||
cancel.cancel();
|
||||
let (tx, rx) = channel();
|
||||
spawn_build(dir.path().to_path_buf(), 2, cancel, tx).unwrap();
|
||||
let summary = loop {
|
||||
if let Update::Ready(summary) = rx.recv_timeout(Duration::from_secs(30)).unwrap() {
|
||||
break summary;
|
||||
}
|
||||
};
|
||||
assert!(summary.cancelled);
|
||||
assert!(summary.files < project.indexed_files);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn updater_follows_disk_and_buffers() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let root = dir.path();
|
||||
write(root, ".gitignore", "*.log\n/ignored/\n");
|
||||
write(root, "src/A.php", "<?php class Alpha {}");
|
||||
let (mut index, _, _) = build_index(root, 2);
|
||||
assert_eq!(index.find_classes("Alpha", 5).len(), 1);
|
||||
let (tx, rx) = channel();
|
||||
let updater = Updater::spawn(root.to_path_buf(), CancelToken::new(), tx).unwrap();
|
||||
|
||||
write(root, "src/A.php", "<?php class Alpha {} class Beta {}");
|
||||
updater.send(Job::File {
|
||||
path: "src/A.php".into(),
|
||||
text: None,
|
||||
});
|
||||
apply_until_quiet(&mut index, &rx);
|
||||
assert_eq!(index.get("src/A.php").unwrap().symbols.len(), 2);
|
||||
|
||||
let buffer = lutra_text::Buffer::new("<?php class Gamma { public function go() {} }");
|
||||
for _ in 0..5 {
|
||||
updater.send(Job::File {
|
||||
path: "src/A.php".into(),
|
||||
text: Some(buffer.clone()),
|
||||
});
|
||||
}
|
||||
apply_until_quiet(&mut index, &rx);
|
||||
assert!(index.find_classes("Beta", 5).is_empty());
|
||||
assert_eq!(index.find_symbols("Gamma::go", 5).len(), 1);
|
||||
|
||||
write(root, "src/sub/deep/B.php", "<?php class Delta {}");
|
||||
write(root, "src/sub/notes.log", "ignored");
|
||||
updater.send(Job::Dir {
|
||||
path: "src/sub".into(),
|
||||
});
|
||||
apply_until_quiet(&mut index, &rx);
|
||||
assert!(index.contains("src/sub/deep/B.php"));
|
||||
assert!(!index.contains("src/sub/notes.log"));
|
||||
|
||||
write(root, "ignored/C.php", "<?php class Epsilon {}");
|
||||
updater.send(Job::File {
|
||||
path: "ignored/C.php".into(),
|
||||
text: None,
|
||||
});
|
||||
apply_until_quiet(&mut index, &rx);
|
||||
assert!(!index.contains("ignored/C.php"));
|
||||
|
||||
fs::remove_dir_all(root.join("src/sub")).unwrap();
|
||||
updater.send(Job::Dir {
|
||||
path: "src/sub".into(),
|
||||
});
|
||||
apply_until_quiet(&mut index, &rx);
|
||||
assert!(!index.contains("src/sub/deep/B.php"));
|
||||
|
||||
fs::remove_file(root.join("src/A.php")).unwrap();
|
||||
updater.send(Job::File {
|
||||
path: "src/A.php".into(),
|
||||
text: None,
|
||||
});
|
||||
apply_until_quiet(&mut index, &rx);
|
||||
assert!(!index.contains("src/A.php"));
|
||||
assert_eq!(index.symbol_count(), 0);
|
||||
}
|
||||
78
crates/index/tests/perf.rs
Normal file
78
crates/index/tests/perf.rs
Normal file
|
|
@ -0,0 +1,78 @@
|
|||
//! Index timing on a generated 20,000-file project. Ignored by default; run
|
||||
//! `cargo test --release -p lutra-index --test perf -- --ignored --nocapture`.
|
||||
|
||||
#![allow(
|
||||
clippy::unwrap_used,
|
||||
clippy::expect_used,
|
||||
clippy::indexing_slicing,
|
||||
clippy::panic
|
||||
)]
|
||||
|
||||
use std::time::Instant;
|
||||
|
||||
use lutra_index::{CancelToken, Index, Update, build, default_threads};
|
||||
|
||||
#[test]
|
||||
#[ignore = "measurement; run in release"]
|
||||
fn index_20k_files() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let project = lutra_testkit::generate_php_project(dir.path(), 20_000);
|
||||
println!(
|
||||
"generated: {} indexed files, {} PHP ({} vendor), {:.1} MB PHP, {} classes",
|
||||
project.indexed_files,
|
||||
project.php_files,
|
||||
project.vendor_files,
|
||||
project.php_bytes as f64 / 1e6,
|
||||
project.classes
|
||||
);
|
||||
let threads = default_threads();
|
||||
for round in 0..3 {
|
||||
let index = std::sync::Mutex::new(Index::new());
|
||||
let mut first_batch = None;
|
||||
let started = Instant::now();
|
||||
let first = std::sync::Mutex::new(&mut first_batch);
|
||||
let summary = build(dir.path(), threads, &CancelToken::new(), &|u| {
|
||||
if let Update::Files { files, .. } = u {
|
||||
let mut first = first.lock().unwrap();
|
||||
if first.is_none() {
|
||||
**first = Some(started.elapsed());
|
||||
}
|
||||
let mut index = index.lock().unwrap();
|
||||
for f in files {
|
||||
index.insert(f);
|
||||
}
|
||||
}
|
||||
});
|
||||
let index = index.into_inner().unwrap();
|
||||
assert_eq!(index.len(), project.indexed_files);
|
||||
let time = |f: &dyn Fn() -> usize| {
|
||||
let mut runs: Vec<f64> = (0..7)
|
||||
.map(|_| {
|
||||
let t = Instant::now();
|
||||
assert!(f() > 0);
|
||||
t.elapsed().as_secs_f64() * 1e3
|
||||
})
|
||||
.collect();
|
||||
runs.sort_by(f64::total_cmp);
|
||||
(runs[0], runs[3])
|
||||
};
|
||||
let class = time(&|| index.find_classes("Serv12", 50).len());
|
||||
let symbol = time(&|| index.find_symbols("handle1", 50).len());
|
||||
let member = time(&|| index.find_symbols("Service12::hand", 50).len());
|
||||
let file = time(&|| index.find_files("wid12", 50).len());
|
||||
println!(
|
||||
"round {round}: {threads} workers, walk {:.0} ms, first batch {:.0} ms, total {:.0} ms, {} files, {} symbols, {} parsed ({:.1} MB)",
|
||||
summary.walk.as_secs_f64() * 1e3,
|
||||
first_batch.unwrap().as_secs_f64() * 1e3,
|
||||
summary.elapsed.as_secs_f64() * 1e3,
|
||||
summary.files,
|
||||
summary.symbols,
|
||||
summary.parsed,
|
||||
summary.parsed_bytes as f64 / 1e6,
|
||||
);
|
||||
println!(
|
||||
" queries (min / median of 7): goto.class {:.1} / {:.1} ms, goto.symbol {:.1} / {:.1} ms, Class::member {:.1} / {:.1} ms, goto.file {:.1} / {:.1} ms",
|
||||
class.0, class.1, symbol.0, symbol.1, member.0, member.1, file.0, file.1
|
||||
);
|
||||
}
|
||||
}
|
||||
88
crates/index/tests/properties.rs
Normal file
88
crates/index/tests/properties.rs
Normal file
|
|
@ -0,0 +1,88 @@
|
|||
//! Property tests: random queries and random file contents never panic, and
|
||||
//! query results respect the limit and the ranking order.
|
||||
|
||||
#![allow(
|
||||
clippy::unwrap_used,
|
||||
clippy::expect_used,
|
||||
clippy::indexing_slicing,
|
||||
clippy::panic
|
||||
)]
|
||||
|
||||
use lutra_index::{Index, IndexedFile, index_text};
|
||||
use lutra_syntax::Parser;
|
||||
use proptest::prelude::*;
|
||||
|
||||
/// A small index with project and library files, unicode names included.
|
||||
fn sample() -> Index {
|
||||
let mut parser = Parser::new().unwrap();
|
||||
let mut index = Index::new();
|
||||
let files = [
|
||||
(
|
||||
"src/Models/User.php",
|
||||
"<?php namespace App\\Models; class User { public function find() {} private $név; const MAX = 1; }",
|
||||
),
|
||||
(
|
||||
"src/Ünïcode/Grüße.php",
|
||||
"<?php namespace Ü; final class Grüße { public function sägen() {} }",
|
||||
),
|
||||
(
|
||||
"src/helpers.php",
|
||||
"<?php function format_user() {} const LIMIT = 3;",
|
||||
),
|
||||
(
|
||||
"vendor/acme/User.php",
|
||||
"<?php namespace Acme; interface User { public function find(); } enum E: int { case A = 1; }",
|
||||
),
|
||||
("templates/a😀b.twig", ""),
|
||||
];
|
||||
for (path, text) in files {
|
||||
index.insert(index_text(&mut parser, path.into(), text));
|
||||
}
|
||||
index
|
||||
}
|
||||
|
||||
proptest! {
|
||||
#![proptest_config(ProptestConfig::with_cases(512))]
|
||||
|
||||
#[test]
|
||||
fn random_queries_never_panic(query in "\\PC{0,40}|[\\\\/:$ _a-zA-Z]{0,12}", limit in 0usize..2_000) {
|
||||
let index = sample();
|
||||
let files = index.find_files(&query, limit);
|
||||
let classes = index.find_classes(&query, limit);
|
||||
let symbols = index.find_symbols(&query, limit);
|
||||
let cap = limit.min(lutra_index::MAX_LIMIT);
|
||||
prop_assert!(files.len() <= cap && classes.len() <= cap && symbols.len() <= cap);
|
||||
for pair in files.windows(2) {
|
||||
prop_assert!(pair[0].library <= pair[1].library);
|
||||
if pair[0].library == pair[1].library {
|
||||
prop_assert!(pair[0].score >= pair[1].score);
|
||||
}
|
||||
}
|
||||
for hit in &files {
|
||||
let len: usize = hit.path.encode_utf16().count();
|
||||
for range in &hit.matches {
|
||||
prop_assert!(range[0] < range[1] && range[1] <= len);
|
||||
}
|
||||
prop_assert!(hit.matches.windows(2).all(|w| w[0][1] < w[1][0]));
|
||||
}
|
||||
prop_assert!(classes.iter().all(|h| h.symbol.kind.is_class_like()));
|
||||
prop_assert!(symbols.iter().all(|h| !h.symbol.kind.is_class_like()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn random_sources_never_panic(bytes in proptest::collection::vec(any::<u8>(), 0..2_000)) {
|
||||
let mut parser = Parser::new().unwrap();
|
||||
let mut text = String::from("<?php ");
|
||||
text.push_str(&String::from_utf8_lossy(&bytes));
|
||||
let file = index_text(&mut parser, "src/x.php".into(), &text);
|
||||
let mut index = Index::new();
|
||||
index.insert(file.clone());
|
||||
for symbol in &file.symbols {
|
||||
prop_assert!(!symbol.name.is_empty());
|
||||
let _ = index.find_symbols(&symbol.name, 10);
|
||||
let _ = index.find_classes(&symbol.name, 10);
|
||||
}
|
||||
index.insert(IndexedFile { path: "src/x.php".into(), library: false, symbols: Vec::new() });
|
||||
prop_assert_eq!(index.symbol_count(), 0);
|
||||
}
|
||||
}
|
||||
|
|
@ -10,6 +10,7 @@
|
|||
clippy::indexing_slicing
|
||||
)]
|
||||
|
||||
pub mod php;
|
||||
pub mod tree;
|
||||
|
||||
use std::path::{Path, PathBuf};
|
||||
|
|
@ -19,6 +20,7 @@ use lutra_control::{Backend, EventSink, Subscribers};
|
|||
use lutra_engine::{CommandInfo, Effect, Engine, EngineConfig, EngineError};
|
||||
use serde_json::Value;
|
||||
|
||||
pub use php::{PhpProject, generate_php_project};
|
||||
pub use tree::{NEEDLE, SearchTree, generate_search_tree};
|
||||
|
||||
/// Engine directories inside `root` (`root/config`, `root/data`), for
|
||||
|
|
|
|||
176
crates/testkit/src/php.rs
Normal file
176
crates/testkit/src/php.rs
Normal file
|
|
@ -0,0 +1,176 @@
|
|||
//! Generated PHP projects for index tests and measurements.
|
||||
//!
|
||||
//! [`generate_php_project`] writes a deterministic Composer-style project of
|
||||
//! any size, almost all PHP: classes, interfaces, traits, enums and function
|
||||
//! files in `src/`, library classes in `vendor/` (gitignored, indexed as
|
||||
//! library code), a few templates and config files, an ignored `var/cache/`
|
||||
//! and an excluded `node_modules/`. Class files are about 5 KB, the size of
|
||||
//! typical framework classes, so 20,000 files are about 100 MB of PHP.
|
||||
|
||||
use std::fs;
|
||||
use std::path::Path;
|
||||
|
||||
/// What [`generate_php_project`] wrote.
|
||||
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
|
||||
pub struct PhpProject {
|
||||
/// Files the index lists (everything but `var/` and `node_modules/`).
|
||||
pub indexed_files: usize,
|
||||
/// PHP files the index parses.
|
||||
pub php_files: usize,
|
||||
/// Of those, files under `vendor/`.
|
||||
pub vendor_files: usize,
|
||||
/// Classes, interfaces, traits and enums declared in indexed files.
|
||||
pub classes: usize,
|
||||
/// Bytes of indexed PHP.
|
||||
pub php_bytes: u64,
|
||||
}
|
||||
|
||||
/// Methods per generated class.
|
||||
const METHODS: usize = 14;
|
||||
|
||||
/// Writes a project of about `files` files below `root`.
|
||||
///
|
||||
/// Shares: 80 % PHP in `src/` (one in ten an interface, one in twenty a
|
||||
/// trait, one in twenty an enum, one in fifty a file of functions, the rest
|
||||
/// classes), 15 % PHP in `vendor/acme/framework/src/`, 3 % Twig templates,
|
||||
/// 2 % YAML config; plus `var/cache/` (ignored) and `node_modules/`
|
||||
/// (excluded) with a few PHP files each that the index must skip.
|
||||
///
|
||||
/// Class `src/Module{m}/Service{i}.php` declares `App\Module{m}\Service{i}`
|
||||
/// with methods `handle0`…, properties `$items` and `$name`, and constant
|
||||
/// `MARKER`; vendor class `i` is `Acme\Framework\Component{m}\Widget{i}`.
|
||||
///
|
||||
/// # Panics
|
||||
/// If a file cannot be written (test helper).
|
||||
pub fn generate_php_project(root: &Path, files: usize) -> PhpProject {
|
||||
let mut out = PhpProject::default();
|
||||
write(root, ".gitignore", "/var/\n/vendor/\n", &mut out, false);
|
||||
out.indexed_files += 1;
|
||||
let src = files.saturating_mul(80) / 100;
|
||||
for i in 0..src {
|
||||
let module = i / 100;
|
||||
let (name, text) = src_file(i, module);
|
||||
let rel = format!("src/Module{module:03}/{name}.php");
|
||||
write(root, &rel, &text, &mut out, true);
|
||||
if i % 50 != 7 {
|
||||
out.classes += 1;
|
||||
}
|
||||
}
|
||||
let vendor = files.saturating_mul(15) / 100;
|
||||
for i in 0..vendor {
|
||||
let module = i / 100;
|
||||
let rel = format!("vendor/acme/framework/src/Component{module:03}/Widget{i}.php");
|
||||
let text = class_text(
|
||||
&format!("Acme\\Framework\\Component{module:03}"),
|
||||
&format!("Widget{i}"),
|
||||
"class",
|
||||
);
|
||||
write(root, &rel, &text, &mut out, true);
|
||||
out.vendor_files += 1;
|
||||
out.classes += 1;
|
||||
}
|
||||
for i in 0..files.saturating_mul(3) / 100 {
|
||||
let rel = format!("templates/page{i}.html.twig");
|
||||
write(
|
||||
root,
|
||||
&rel,
|
||||
"{% block body %}<p>{{ content }}</p>{% endblock %}\n",
|
||||
&mut out,
|
||||
false,
|
||||
);
|
||||
out.indexed_files += 1;
|
||||
}
|
||||
for i in 0..files.saturating_mul(2) / 100 {
|
||||
let rel = format!("config/packages/package{i}.yaml");
|
||||
write(
|
||||
root,
|
||||
&rel,
|
||||
&format!("package{i}:\n enabled: true\n"),
|
||||
&mut out,
|
||||
false,
|
||||
);
|
||||
out.indexed_files += 1;
|
||||
}
|
||||
for i in 0..3 {
|
||||
let text = class_text("Cache", &format!("Cached{i}"), "class");
|
||||
fs::create_dir_all(root.join("var/cache")).unwrap();
|
||||
fs::write(root.join(format!("var/cache/Cached{i}.php")), &text).unwrap();
|
||||
fs::create_dir_all(root.join("node_modules/pkg")).unwrap();
|
||||
fs::write(root.join(format!("node_modules/pkg/Hidden{i}.php")), &text).unwrap();
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// Name and text of the `i`th source file in module `module`.
|
||||
fn src_file(i: usize, module: usize) -> (String, String) {
|
||||
let namespace = format!("App\\Module{module:03}");
|
||||
match i % 50 {
|
||||
7 => (
|
||||
format!("functions{i}"),
|
||||
format!(
|
||||
"<?php\n\nnamespace {namespace};\n\nfunction helper{i}(int $id): string\n{{\n return (string) $id;\n}}\n\nfunction format{i}(string $text): string\n{{\n return trim($text);\n}}\n\nconst LIMIT{i} = {i};\n"
|
||||
),
|
||||
),
|
||||
n if n % 10 == 3 => (
|
||||
format!("Contract{i}"),
|
||||
class_text(&namespace, &format!("Contract{i}"), "interface"),
|
||||
),
|
||||
n if n % 20 == 5 => (
|
||||
format!("Helps{i}"),
|
||||
class_text(&namespace, &format!("Helps{i}"), "trait"),
|
||||
),
|
||||
n if n % 20 == 15 => (
|
||||
format!("Status{i}"),
|
||||
enum_text(&namespace, &format!("Status{i}")),
|
||||
),
|
||||
_ => (
|
||||
format!("Service{i}"),
|
||||
class_text(&namespace, &format!("Service{i}"), "class"),
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
/// A class, interface or trait of about 5 KB.
|
||||
fn class_text(namespace: &str, name: &str, keyword: &str) -> String {
|
||||
let mut s = format!(
|
||||
"<?php\n\ndeclare(strict_types=1);\n\nnamespace {namespace};\n\nuse Psr\\Log\\LoggerInterface;\nuse App\\Contracts\\Handler;\n\n/**\n * {name} handles generated work for measurements.\n */\n{keyword} {name}\n{{\n"
|
||||
);
|
||||
if keyword != "interface" {
|
||||
s.push_str(" public const MARKER = 'plain';\n\n private array $items = [];\n\n protected ?string $name = null;\n\n");
|
||||
}
|
||||
for m in 0..METHODS {
|
||||
if keyword == "interface" {
|
||||
s.push_str(&format!(
|
||||
" /**\n * Handles step {m}.\n */\n public function handle{m}(int $id, string $name): ?string;\n\n"
|
||||
));
|
||||
continue;
|
||||
}
|
||||
s.push_str(&format!(
|
||||
" /**\n * Handles step {m} of the generated workflow.\n */\n public function handle{m}(int $id, string $name): ?string\n {{\n $key = sprintf('%s-%d', $name, $id + {m});\n if (isset($this->items[$key])) {{\n return $this->items[$key];\n }}\n $this->items[$key] = strtoupper($name);\n return null;\n }}\n\n"
|
||||
));
|
||||
}
|
||||
s.push_str("}\n");
|
||||
s
|
||||
}
|
||||
|
||||
/// A backed enum with a few cases and a method.
|
||||
fn enum_text(namespace: &str, name: &str) -> String {
|
||||
format!(
|
||||
"<?php\n\nnamespace {namespace};\n\nenum {name}: string\n{{\n case Draft = 'draft';\n case Published = 'published';\n case Archived = 'archived';\n\n public function label(): string\n {{\n return ucfirst($this->value);\n }}\n}}\n"
|
||||
)
|
||||
}
|
||||
|
||||
/// Writes `text` to `root/rel`, creating directories, and counts it as an
|
||||
/// indexed PHP file if `php`.
|
||||
fn write(root: &Path, rel: &str, text: &str, out: &mut PhpProject, php: bool) {
|
||||
let path = root.join(rel);
|
||||
if let Some(parent) = path.parent() {
|
||||
fs::create_dir_all(parent).unwrap();
|
||||
}
|
||||
fs::write(path, text).unwrap();
|
||||
if php {
|
||||
out.indexed_files += 1;
|
||||
out.php_files += 1;
|
||||
out.php_bytes += text.len() as u64;
|
||||
}
|
||||
}
|
||||
|
|
@ -1,6 +1,6 @@
|
|||
# Architecture
|
||||
|
||||
> **Status:** M0 decisions recorded, M1 workspace, desktop bridge, tabs and editor, message catalog, engine construction, settings, themes, the recovery journal, notifications, UI commands, the palette, the settings dialog, the app icon and the syntax layer documented; M2 highlighting and `editor.reveal`, plus the search backend and the Find tool window · **Last updated:** 2026-10-17
|
||||
> **Status:** M0 decisions recorded, M1 workspace, desktop bridge, tabs and editor, message catalog, engine construction, settings, themes, the recovery journal, notifications, UI commands, the palette, the settings dialog, the app icon and the syntax layer documented; M2 highlighting and `editor.reveal`, plus the search backend, the Find tool window and the symbol index · **Last updated:** 2026-10-17
|
||||
|
||||
## Overview
|
||||
|
||||
|
|
@ -243,6 +243,19 @@ Watched directories: the VFS watcher is non-recursive, so `project.children` wat
|
|||
- **Window:** `ui/src/components/find/FindToolWindow.vue`. Enter in a field searches, a toggle repeats the last search, cancel while running. The status line shows counts while streaming, then matches, files and elapsed time, "No matches" or "Cancelled"; a truncated search adds a hint. Results are a listbox with `aria-activedescendant`: file headers (name, directory, count) and one option per match (1-based line number, preview without leading indentation, the match in a `<mark>` with `--lt-editor-match-bg`). Down in a field enters the list; Up/Down, Page Up/Down (10), Home, End move; Enter or a click runs `editor.reveal {path, line, column}` through the command store. Every match is rendered; with the default cap of 10,000 that stays a plain DOM list (virtual scrolling is a follow-up if it shows up as slow).
|
||||
- **`find.open`** (UI command, `Mod+Alt+F`, Lutra's own; provided by `MainLayout.vue`, so it works before the panel exists): shows and expands the panel and focuses and selects the query field. Without params it takes the main selection (single line, at most 1,000 characters) or else the word at the cursor of the active editor state (`prefillFrom`), and waits for Enter. With a string `query` (and optional `regex`, `caseSensitive`, `wholeWord`, `include`, `exclude`), as an agent sends it through `uiCommand`, it searches at once. The panel is a tool window, not a modal: it is not part of the modal rule.
|
||||
|
||||
### Index and go-to (M2.5)
|
||||
|
||||
- `crates/index` (L2, depends on `lutra-syntax`, `lutra-text`, `ignore` and `nucleo-matcher` 0.3, MPL-2.0 as a separate crate, built without its `unicode-segmentation` feature) lists every file of the project and the declarations of its PHP files. A `Symbol` is `{name, fqn, kind, container, line, column}`: names as written (`$id` for properties), FQNs without a leading `\` (`App\Models\User`, members `App\Models\User::find`), `container` the declaring class-like FQN, the position of the name (0-based line, UTF-16 column) from the outline's `selectionRange`. Namespaces only prefix FQNs; anonymous classes and their members are skipped; at most 20,000 symbols per file.
|
||||
- Which files: the walk is configured to the same rules as `crates/search` (ignore files without a git repository, `.git` and `node_modules` skipped, the root's `vendor/` walked as a second root even when gitignored and marked `library`, dot files listed, links not followed). Every file is listed for Go to File; only `.php` files up to 2 MiB (`MAX_PARSE_BYTES`) are parsed, larger ones are listed without symbols. At most 200,000 files (`MAX_FILES`); further files are left out (`Summary::capped`).
|
||||
- Build: `build`/`spawn_build` walk first (parallel `ignore` walker), report non-PHP files at once, then parse PHP files on `default_threads()` workers (cores − 1, at most 32), each with its own `Parser` and one file in memory, taking files from a shared counter: project files before library files, larger files first in each group. Workers send `Update::Files` batches of 64 files (or every 100 ms) and `Update::Progress {done, total}`; the build ends with `Update::Ready(Summary)`. A panic while parsing a file (a parser bug) is caught per file: the file is listed without symbols and the worker gets a fresh parser. Reads are capped, invalid UTF-8 is replaced, a BOM dropped.
|
||||
- Incremental: `Updater` is one background thread taking `Job::File {path, text?}` (re-index from a buffer's text, or re-read from disk; a file no longer found or now excluded is reported `Removed`) and `Job::Dir {path}` (drop the subtree, list it again with a walk restricted to the path, so ignore files on the way still apply). Jobs queued while it works are coalesced to the last per path, so a burst of keystrokes parses a buffer once.
|
||||
- Queries: `Index` (a `BTreeMap` by path, so a directory's files drop with one range) answers `find_files`, `find_classes` and `find_symbols`. Every whitespace-separated query word must match fuzzily (fzf-style scoring, smart case, no other special syntax); queries are capped at 256 characters and 1,000 results. Files: a query without `/` (a `\` counts as `/`) is matched against the file name first with double weight, else the path; match ranges are merged UTF-16 ranges in the path. Classes: by name, by FQN when the query has a `\`. Symbols: by name; `Class::member` matches the container's short name (FQN with a `\`) and the member separately, `Class::` lists members. Order: project before library, higher score, shorter name, byte order. Indexes over 20,000 files plus symbols are scanned on `default_threads()` threads, each keeping its own best hits.
|
||||
- Engine (`cmd/goto.rs`): `State::index` holds the index of the open project. `goto::sync` runs on every `Engine::pump_vfs` and before every index command: it starts the build and the updater when a project is open and the index belongs to another root (or none), drops them without a project (dropping cancels the threads; `project.close` also drops it at once), merges up to 4,096 updates, emits `indexProgress {done, total}` at most every 250 ms while building and `indexReady {files, symbols, elapsedMs}` at the end, and re-indexes buffers. The engine lock is held only for inserts and queries; walking and parsing never run under it. Updater results that arrive while the build runs mark their paths, and later build results for those paths are ignored, so a buffer edit or a disk change is never overwritten by the stale build.
|
||||
- Incremental sources: `workspace::pump` hands the VFS events to `goto::note_disk_changes` (removed paths dropped at once, created or modified files re-read, created directories and rename targets rescanned); the watcher is not recursive, so only watched directories (root, listed, pinned) report changes. A buffer whose version changed is re-indexed from its text when its path is in the index (or the build still runs); while it is dirty, disk changes for it are ignored; closing it dirty re-reads the file.
|
||||
- Commands: `index.status` → `{state: "idle"|"indexing", files, symbols}` (idle and zero without a project). `goto.file {query, limit?}` → `{items: [{path, score, matches: [[start, end]]}], indexing}`; `goto.class` → `{items: [{name, fqn, kind, path, line, column, library}], indexing}`; `goto.symbol` → the same plus `container` (FQN or `null`), for functions, methods, properties, constants and enum cases. `limit` defaults to 50 (at most 1,000); an empty query returns no items. The go-to commands are disabled without a project; none has a menu path or shortcut yet (M2.6 adds the popup).
|
||||
- Measured (release, Windows 11, 8 cores / 16 threads, 15 workers, generated `lutra_testkit::generate_php_project(20_000)`: 20,001 files, 19,000 PHP of ~5 KB, 87.7 MB, 324,080 symbols): build 3.9–4.2 s on an idle machine (target < 5 s met), 4.6–5.7 s while other agents' builds ran; the walk takes 25–35 ms and the first batch arrives after ~30 ms, so the parse dominates (~21 MB/s over 15 workers, ~1.4 MB/s per worker). Queries on the full index, median of 7: `goto.class` 2–4 ms, `goto.symbol` 7–11 ms, `Class::member` 11–12 ms, `goto.file` 2 ms (single-threaded before the parallel scan: `goto.symbol` 72–100 ms). `cargo test --release -p lutra-index --test perf -- --ignored --nocapture` repeats it.
|
||||
- Live check over the control channel against the running app (debug build, port 7825, generated 6,002-file project with gitignored `vendor/` and `var/`): `index.status` answered `indexing` right after `project.open`, a go-to query during the build returned `indexing: true`, 25 `indexProgress` events at least 251 ms apart, `indexReady` after 8.2 s (debug build: the tree-sitter C code and the workers are unoptimised) with 6,002 files and 62,000 symbols; `goto.file`, `goto.class` (vendor marked `library`), `goto.symbol` with `Class::member` answered in 5–25 ms; the gitignored `var/` class was absent; a `buffer.edit` renaming a class and a file created on disk were found within a second; `project.close` reset the status to idle.
|
||||
|
||||
## Decisions (M0)
|
||||
|
||||
| Question | Decision | Why |
|
||||
|
|
@ -260,6 +273,7 @@ Watched directories: the VFS watcher is non-recursive, so `project.children` wat
|
|||
|
||||
| Date | Change | Summary |
|
||||
|---|---|---|
|
||||
| 2026-10-17 | minor | M2.5: symbol index (`crates/index`: parallel build, updater, fuzzy queries), `index.status`, `goto.file`/`goto.class`/`goto.symbol`, `indexProgress`/`indexReady` via the pump, timing |
|
||||
| 2026-10-17 | minor | M2.4: bottom panel, Find tool window (store, event race, one search at a time), `find.open` |
|
||||
| 2026-10-16 | minor | M2.2: highlighting by file extension (Lezer), `--lt-syntax-*` tokens and classes, `editor.reveal`, keystroke-to-highlight measurement |
|
||||
| 2026-10-16 | minor | M2.3: find in files backend (`crates/search`, `search.find`/`search.cancel`, `searchResults`/`searchDone` via the pump) |
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
# Gaps
|
||||
|
||||
> **Status:** M0 closed G1; G2 under way (text buffer, engine buffers, tabs and editor in the app, recovery journal); G3 M1 part done (project tree, palette, settings dialog, themes); G4 under way (tree-sitter outline and folding library, Lezer highlighting); G5 under way (search backend, Find tool window); G14 English UI catalog in place · **Last updated:** 2026-10-17
|
||||
> **Status:** M0 closed G1; G2 under way (text buffer, engine buffers, tabs and editor in the app, recovery journal); G3 M1 part done (project tree, palette, settings dialog, themes); G4 under way (tree-sitter outline and folding library, Lezer highlighting); G5 under way (search backend, Find tool window, symbol index and go-to commands); G14 English UI catalog in place · **Last updated:** 2026-10-17
|
||||
|
||||
Every known shortfall, ranked. This is the work list: unless the owner gives a task, take the highest open gap. Hours are agent wall-clock hours for one agent, all estimated.
|
||||
|
||||
|
|
@ -31,7 +31,7 @@ G11 progress: M1.7 added the crash-recovery journal: unsaved buffers are written
|
|||
|
||||
G14 progress: M1.11 moved every UI string in `ui/src` into the English catalog `ui/src/i18n/en.json` behind `t(key, params?)`, enforced by `vue/no-bare-strings-in-template` and a catalog test. Messages the engine sends (`EngineError.message`, the slogan from `app.about`) are still English text; they need message keys with params before a second language can cover them.
|
||||
|
||||
G5 progress: M2.3 landed the find-in-files backend: `crates/search` (parallel walk with the project's ignore rules, literal or regex, case, whole word, globs, binary and size skips, match cap, cancel) and the engine commands `search.find` / `search.cancel` with streamed `searchResults` and `searchDone` events; dirty buffers are searched in their buffer text. First result on a generated 20k-file tree 10–43 ms (release, measured). M2.4 added the Find tool window in a new bottom panel (options, globs, grouped and highlighted results, counts, cancel, truncated hint, keyboard navigation, `editor.reveal`, `find.open` `Mod+Alt+F` with prefill). Open in G5: scopes, history and replace in files, the symbol index and go-to (M2.5, M2.6).
|
||||
G5 progress: M2.3 landed the find-in-files backend: `crates/search` (parallel walk with the project's ignore rules, literal or regex, case, whole word, globs, binary and size skips, match cap, cancel) and the engine commands `search.find` / `search.cancel` with streamed `searchResults` and `searchDone` events; dirty buffers are searched in their buffer text. First result on a generated 20k-file tree 10–43 ms (release, measured). M2.4 added the Find tool window in a new bottom panel (options, globs, grouped and highlighted results, counts, cancel, truncated hint, keyboard navigation, `editor.reveal`, `find.open` `Mod+Alt+F` with prefill). M2.5 added the first symbol index (`crates/index`: every project file, PHP declarations from tree-sitter outlines, vendor as library, parallel build off the engine lock, incremental on disk and buffer changes, cancelled on project close) and the commands `index.status`, `goto.file`, `goto.class`, `goto.symbol` with `indexProgress`/`indexReady` events; a generated 20k-file project (88 MB of PHP) indexes in 3.9–4.2 s (release, measured, target 5 s), queries take 2–12 ms. Open in G5: replace in files, scopes and history, the go-to popup (M2.6), an index that survives restarts, changes in directories the non-recursive watcher does not watch, usages.
|
||||
|
||||
Known smaller items from M0:
|
||||
|
||||
|
|
@ -43,6 +43,7 @@ Known smaller items from M0:
|
|||
|
||||
| Date | Change | Summary |
|
||||
|---|---|---|
|
||||
| 2026-10-17 | minor | G5: symbol index and go-to commands (M2.5) |
|
||||
| 2026-10-17 | minor | G5: Find tool window (M2.4) |
|
||||
| 2026-10-16 | minor | G4: Lezer highlighting and `editor.reveal` (M2.2) |
|
||||
| 2026-10-16 | minor | G5: find-in-files backend (M2.3) |
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
# Target-app parity
|
||||
|
||||
> **Status:** M1 done (text buffer, VFS, engine workspace, project tree, tabs and a plain-text editor, recovery, palette, settings and themes in the app); M2 started: PHP syntax highlighting, find in files with its tool window · **Last updated:** 2026-10-17 · **Target:** PhpStorm 2026.x
|
||||
> **Status:** M1 done (text buffer, VFS, engine workspace, project tree, tabs and a plain-text editor, recovery, palette, settings and themes in the app); M2 started: PHP syntax highlighting, find in files with its tool window, the symbol index and go-to commands · **Last updated:** 2026-10-17 · **Target:** PhpStorm 2026.x
|
||||
|
||||
## How this is measured
|
||||
|
||||
|
|
@ -14,7 +14,7 @@ Hours are the estimated agent wall-clock hours for one agent to reach behaviour
|
|||
|---|---:|---:|---|---:|---|
|
||||
| Editor core | 12 | 10% (est.) | semantic highlighting and a colour scheme editor, code folding, soft wrap and other editor settings, preview tab, pinned/split/reordered tabs, close others/all, reopen closed tab, tab limit, conflict UI for files changed on disk, autosave, local history, encodings other than UTF-8, large-file mode, column selection and multi-caret checked against the docs; recovery journal not yet checked in the running app (gaming mode) | ~104 (est.) | M2.2: syntax highlighting by file extension (`ui/src/components/editor/language.ts`): PHP with embedded HTML/CSS/JS through `@codemirror/lang-php` (Lezer), plus HTML, CSS, JS/TS, JSON, YAML, Markdown; spans carry `lt-syntax-*` classes coloured only by the eleven `--lt-syntax-*` tokens of both themes (Lutra's own palette, ≥ 4.5:1 on the editor background and active line, contrast test). `editor.reveal {path, line, column}` opens, activates, places the cursor (UTF-16 column, CRLF-safe) and centres it; agents reach it as a UI command. Checked in the app on Windows 11 (bundled UI, control channel): PHP with embedded HTML in dark and light, `editor.reveal` into a closed CRLF file (last line, then line 302 centred). Vitest: language per extension, classes on spans for PHP/HTML and the other languages, reveal on open and closed files, CRLF mapping, clamping, bad params. keystroke to highlight on a 1 MB PHP file 3.0–11.4 ms median (jsdom, JavaScript work only, measured, not yet in the WebView). M2.1: `crates/syntax` parses PHP with tree-sitter (incremental reparse from `lutra_text` edits) and computes folding ranges (brace blocks, arrays, comments, heredocs, `use` runs); library only, nothing visible in the editor yet, so the score is unchanged. Tests: outline/folding snapshots for five PHP 8.3 fixtures, proptests (random bytes and token soups never panic, reparse equals a fresh parse, undo round trip). 1 MB parse 364–553 ms, reparse after one keystroke 7–11 ms (release, measured, machine busy). M1.13: opening a file that is already open switches to its tab (`bufferFocused`; engine test, Vitest, control-channel test; live check pending (gaming mode)); files in folders the tree never expanded still reload or report changes on disk, because the parent directory of every open buffer stays watched (engine tests). M1.7: crash-recovery journal (`crates/engine/src/journal.rs`, `cmd/recovery.rs`): dirty buffers are written atomically to `<data_dir>/recovery/<project-id>/` about 1 s after the last edit (at least every 2 s while typing), never per keystroke; removed on save, close and discard; `recoveryAvailable` after `project.open`, `recovery.list/restore/discard`; restored buffers are dirty and keep line ending and BOM; corrupt entries are moved aside with a warning. Restore dialog (`ui/src/components/recovery/`: Restore all / selected / Discard, time, changed-on-disk hint). Tests: engine dropped mid-edit and a new engine restores the exact CRLF + BOM text, corrupt entries, removal paths, a property test of random edit sequences → crash → restore equals the buffer (`crates/engine/tests/recovery.rs`, 10 tests), the same flow headless over the control channel (`crates/control/tests/recovery.rs`), 10 Vitest tests for the dialog. 1 MiB entry write: 8.5 ms median (release, measured). Graded against `plan/phpstorm/04-recovery.md`. M1.6: tabs and a CodeMirror 6 editor in the app (`ui/src/components/editor/`, `ui/src/stores/editor.ts`): open from the project tree or engine events, activate, close by button, middle click or `Mod+W`, dirty dot, Save / Don't Save / Cancel on closing a dirty tab, `Mod+S` / `Mod+Alt+S` from engine command specs, non-blocking notice for a dirty file changed or deleted on disk; local edits go to `buffer.edit` with version-based resync, CRLF files keep `\r\n`; undo/redo, multiple selections, search and bracket matching from CodeMirror's basic setup. Vitest: a burst of 400 keystrokes with stale versions and remote edits loses none, a CRLF burst with deletions converges, store and component tests against a fake engine (`ui/tests/editor/`, 22 tests). Checked in the app on Windows 11 with `--control` and `ui.screenshot`: typing via the UI and via `buffer.edit`, save to disk, close question, middle click, disk-change notice. 1 MB file (16,385 lines): engine `file.open` 80 ms over the control channel (debug build, measured), editor state 1–6 ms and first rendered view 4–11 ms in Chromium (measured), ≈ 90 ms in total (estimated, budget 150 ms). Graded against `plan/phpstorm/03-editor-tabs.md`. `crates/text`: rope buffer, positions, edits (M1.1, unit tests + proptests). M1.3: engine buffers with versions and dirty state, `file.open`, `buffer.get`, `buffer.edit` (UTF-16 change sets, `stale_version`), `file.save`, `file.saveAll`, `buffer.close` with unsaved-changes guard, line ending and BOM kept on save, clean buffers reload on external change; 14 integration tests on temp dirs (`crates/engine/tests/workspace.rs`) |
|
||||
| Project view, VFS | 6 | 8% (est.) | views list (project / scopes), toolbar (Select Opened File, Expand/Collapse All), sort and display options, show/hide excluded, distinct icons for library and excluded folders, file colours, speed search, preview, file operations | ~37 (est.) | `lutra-vfs` (M1.2): lazy sorted listing, ignore rules (.gitignore, .ignore, info/exclude, global excludes, Lutra defaults), native file watching via `notify` with a polling fallback, capped UTF-8 reads, atomic writes; 34 tests on temp dirs, watcher tests run on both backends (`crates/vfs/tests`). M1.5: project tree UI (lazy listing, keyboard navigation, dimmed ignored entries, event refresh) in `ui/src/components/project/`, Vitest against a mocked engine. M1.3: `project.open`, `project.close`, `project.children` (watches listed directories, capped at 256 native / 32 polling, least recently listed dropped), `treeChanged` events from disk, `events.subscribe` on the control channel with a TCP test (`crates/control/tests/events.rs`). M1.4: the tree runs against the engine in the app: folder picker (`project.openDialog`) and `lutra-desktop --control PORT PATH` open a project, the tree lists it lazily with ignored (`var/` from `.gitignore`) and excluded (`node_modules`) entries dimmed, a file created on disk appears in an expanded folder within ~100 ms; verified with `ui.screenshot` on Windows 11 and Vitest (`ui/tests/project/`, `ui/tests/ipcEvents.test.ts`). M1.13: a directory listed only partly (more than 100,000 entries) ends with a "Listing truncated" note row (Vitest). M1.12: a folder that cannot be opened from the picker or the command line (including `unsaved_changes` when switching projects) and stopped file watching show as toasts instead of stderr (`apps/lutra-desktop/src/host.rs`, `ui/src/stores/notifications.ts`, `ui/tests/notifications/`, 19 Vitest tests; live check pending, gaming mode); score unchanged. Graded against `plan/phpstorm/02-project-view.md`: tool window, folder-first tree, dimmed ignored/excluded, keyboard navigation and live refresh exist; most toolbar, view and display options do not |
|
||||
| Navigation | 10 | 0% | everything (M1.8: a command palette over commands only, no go-to-file/class/symbol yet; score unchanged) | ~120 (est.) | M2.1: file outline in `crates/syntax` (namespaces braced and unbraced, classes, interfaces, traits, enums with cases, functions, methods, properties incl. promoted ones, constants; UTF-16 ranges and short signatures), the base for the Structure window (M2.7) and the symbol index (M2.5); not reachable from the UI yet, score unchanged. M1.8: `ui/src/components/palette/`, `ui/src/commands/` (fuzzy ranking, recent commands, global keymap), Vitest `ui/tests/palette/`, `ui/tests/commands/` |
|
||||
| Navigation | 10 | 0% | everything a user can reach: the go-to popup (M2.6), Structure (M2.7), find usages, go to declaration, recent files, navigation history (M1.8: a command palette over commands only; M2.5: go-to backend without UI; score unchanged) | ~112 (est.) | M2.5: symbol index in `crates/index` (every project file listed, `.php` parsed in parallel from tree-sitter outlines, vendor marked library, incremental on disk and buffer changes, capped at 200,000 files) and the engine commands `index.status`, `goto.file`, `goto.class`, `goto.symbol` (fuzzy, file names weighted, `Class::member`, project before library, partial results with `indexing: true`), events `indexProgress`/`indexReady`; 20k-file / 88 MB generated project indexed in 3.9–4.2 s (release, measured), queries 2–12 ms; tests `crates/index/tests/`, `crates/engine/tests/goto.rs`, `crates/control/tests/goto.rs`; not reachable from the UI yet (M2.6). M2.1: file outline in `crates/syntax` (namespaces braced and unbraced, classes, interfaces, traits, enums with cases, functions, methods, properties incl. promoted ones, constants; UTF-16 ranges and short signatures), the base for the Structure window (M2.7) and the symbol index (M2.5); not reachable from the UI yet, score unchanged. M1.8: `ui/src/components/palette/`, `ui/src/commands/` (fuzzy ranking, recent commands, global keymap), Vitest `ui/tests/palette/`, `ui/tests/commands/` |
|
||||
| Search and replace | 6 | 8% (est.) | find in the current file, replace in files and in a file, scopes (module, directory, open files, custom), comment/string filters, search history, a tab per search, grouping options, merging matches on one line, find in path from the tree, structural search | ~70 (est.) | M2.4 Find tool window in a new resizable, collapsible bottom panel: query with case / whole word / regex toggles, include and exclude globs, streamed results grouped by file (sorted) with the match highlighted in the line preview, counts and elapsed time, cancel, a truncated hint, a refused regex or glob shown in place; Enter or click opens the match through `editor.reveal`, arrow keys / Page / Home / End walk the results; `find.open` (`Mod+Alt+F`, Lutra's own) prefills the selection or the word at the cursor, `find.open {query}` from an agent searches at once; a new search cancels the running one. Tests: 26 Vitest store and component tests (`ui/tests/find/FindToolWindow.test.ts`). Live check (bundled-ui debug build, control port 7824, 3-file temp project): `find.open {query}` over `uiCommand` showed 14 matches in 3 files (vendor included) grouped and highlighted in 17 ms; with `*.php` include 2 matches; revealing a result's position opened the file at the match; `find.open` without params prefilled the word at the cursor; a bad regex showed the engine's message in place (screenshots read, measured). Opening a result by mouse or Enter is covered by Vitest only (the control channel cannot click). M2.3 backend: `crates/search` (parallel `ignore` walk + `grep-searcher`: literal/regex, case, whole word, include/exclude globs, project ignore rules with `vendor/` searched and `node_modules` skipped, binary and > 8 MiB files skipped, match cap, cancel) and engine `search.find` / `search.cancel` with `searchResults` / `searchDone` events through the pump; dirty buffers are searched in their buffer text. Tests: 9 library tests incl. a property test of random queries and file bytes (`crates/search/tests/search.rs`), 3 preview/column unit tests, 4 engine tests (`crates/engine/tests/search.rs`), a TCP control-channel test (find → results → done, invalid regex, cancel; `crates/control/tests/search.rs`). 20k-file generated tree, release: first result 10–43 ms, full walk 0.66–1.43 s (measured); live over the control channel against the running app: first result 55 ms, done 386 ms on a 5,900-file project (debug build, measured) |
|
||||
| Code completion | 10 | 0% | everything | ~250 (est.) | — |
|
||||
| Inspections and quick-fixes | 8 | 0% | everything | ~300 (est.) | — |
|
||||
|
|
@ -63,6 +63,7 @@ The project tree is the first user-visible feature (Project view 8%, estimated);
|
|||
|
||||
| Date | Change | Summary |
|
||||
|---|---|---|
|
||||
| 2026-10-17 | minor | M2.5: Navigation evidence (symbol index and go-to commands, no UI), score unchanged at 0%, hours 120 → 112 |
|
||||
| 2026-10-17 | minor | M2.4: Search and replace 0% → 8% (est.), Find tool window; hours 76 → 70 |
|
||||
| 2026-10-16 | minor | M2.2: Editor core 9% → 10% (est.), syntax highlighting and `editor.reveal` |
|
||||
| 2026-10-16 | minor | M2.3: Search and replace evidence (find-in-files backend, no UI), score unchanged at 0%, hours 80 → 76 |
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue