M2.5: symbol index and go-to commands

crates/index: walks the project with the search/VFS rules (vendor as
library), parses .php files from tree-sitter outlines on a worker pool
off the engine lock, updates incrementally from disk and buffers through
an updater thread, caps at 200,000 files, and answers fuzzy file, class
and symbol queries (nucleo-matcher, MPL-2.0 as a separate crate).

Engine: index.status, goto.file, goto.class, goto.symbol; events
indexProgress and indexReady drained by the pump; the index starts on
project.open and is cancelled on close. Tests: unit, incremental,
property (random queries and sources), engine and TCP control tests,
release timing on a generated 20k-file project (3.9-4.2 s).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
danny 2026-10-17 12:53:33 +02:00
parent f13a49064d
commit 053441d066
26 changed files with 3455 additions and 9 deletions

222
crates/index/tests/index.rs Normal file
View file

@ -0,0 +1,222 @@
//! Building the index of a project on disk, incremental updates, and
//! cancellation.
#![allow(
clippy::unwrap_used,
clippy::expect_used,
clippy::indexing_slicing,
clippy::panic
)]
use std::fs;
use std::path::Path;
use std::sync::Mutex;
use std::sync::mpsc::{Receiver, channel};
use std::time::Duration;
use lutra_index::{CancelToken, Index, Job, Update, Updater, build, spawn_build};
/// Builds the index of `root` and applies every update.
fn build_index(root: &Path, threads: usize) -> (Index, lutra_index::Summary, Vec<(usize, usize)>) {
let updates = Mutex::new(Vec::new());
let summary = build(root, threads, &CancelToken::new(), &|u| {
updates.lock().unwrap().push(u)
});
let mut index = Index::new();
let mut progress = Vec::new();
for update in updates.into_inner().unwrap() {
match update {
Update::Files { files, live } => {
assert!(!live);
for file in files {
assert!(index.insert(file));
}
}
Update::Progress { done, total } => progress.push((done, total)),
other => panic!("unexpected {other:?}"),
}
}
(index, summary, progress)
}
/// Applies updates from `rx` until one batch arrives and `rx` is quiet.
fn apply_until_quiet(index: &mut Index, rx: &Receiver<Update>) {
let first = rx.recv_timeout(Duration::from_secs(10)).expect("an update");
let mut pending = vec![first];
while let Ok(next) = rx.recv_timeout(Duration::from_millis(300)) {
pending.push(next);
}
for update in pending {
match update {
Update::Files { files, live } => {
assert!(live);
for file in files {
index.insert(file);
}
}
Update::Removed { path } => {
index.remove(&path);
}
other => panic!("unexpected {other:?}"),
}
}
}
fn write(root: &Path, rel: &str, text: &str) {
let path = root.join(rel);
fs::create_dir_all(path.parent().unwrap()).unwrap();
fs::write(path, text).unwrap();
}
#[test]
fn builds_with_the_project_rules() {
let dir = tempfile::tempdir().unwrap();
let project = lutra_testkit::generate_php_project(dir.path(), 400);
write(
dir.path(),
"src/Broken.php",
"<?php\nclass Broken {\n public function ok() {}\n public function bad() { $x = ; }\n",
);
write(dir.path(), "src/Latin1.php", "<?php\nclass Caf\u{e9} {}\n");
fs::write(
dir.path().join("src/Bad.php"),
b"<?php\nclass Bad\xff {}\nclass Fine {}\n",
)
.unwrap();
write(dir.path(), ".git/Hidden.php", "<?php class InGit {}");
for threads in [1, 4] {
let (index, summary, progress) = build_index(dir.path(), threads);
assert_eq!(index.len(), project.indexed_files + 3);
assert_eq!(summary.files, index.len());
assert_eq!(summary.symbols, index.symbol_count());
assert_eq!(summary.parsed, project.php_files + 3);
assert!(!summary.capped && !summary.cancelled);
let (done, total) = *progress.last().unwrap();
assert_eq!((done, total), (index.len(), index.len()));
assert!(progress.windows(2).all(|w| w[0].0 <= w[1].0));
assert!(index.get("var/cache/Cached0.php").is_none(), "gitignored");
assert!(
index.get("node_modules/pkg/Hidden0.php").is_none(),
"excluded"
);
assert!(index.get(".git/Hidden.php").is_none());
let vendor = index
.get("vendor/acme/framework/src/Component000/Widget0.php")
.unwrap();
assert!(vendor.library, "vendor is walked although gitignored");
let classes = index.find_classes("Service1", 200);
let service = classes
.iter()
.find(|h| h.symbol.fqn == "App\\Module000\\Service1")
.unwrap();
assert_eq!(service.path, "src/Module000/Service1.php");
assert_eq!((service.symbol.line, service.symbol.column), (12, 6));
assert_eq!(index.find_classes("Broken", 5)[0].symbol.name, "Broken");
assert!(!index.find_symbols("Broken::ok", 5).is_empty());
assert_eq!(index.find_classes("Café", 5)[0].symbol.name, "Café");
assert!(!index.find_classes("Fine", 5).is_empty());
let classes_found = index.find_classes("", 10);
assert!(classes_found.is_empty());
let all: usize = index
.find_classes("e", 1_000)
.iter()
.filter(|h| h.symbol.kind.is_class_like())
.count();
assert!(all > 0);
}
}
#[test]
fn spawned_build_ends_with_ready_and_cancels() {
let dir = tempfile::tempdir().unwrap();
let project = lutra_testkit::generate_php_project(dir.path(), 300);
let (tx, rx) = channel();
spawn_build(dir.path().to_path_buf(), 2, CancelToken::new(), tx).unwrap();
let mut files = 0;
let summary = loop {
match rx.recv_timeout(Duration::from_secs(30)).unwrap() {
Update::Files { files: batch, .. } => files += batch.len(),
Update::Ready(summary) => break summary,
_ => {}
}
};
assert_eq!(files, project.indexed_files);
assert_eq!(summary.files, files);
assert!(rx.recv_timeout(Duration::from_millis(200)).is_err());
let cancel = CancelToken::new();
cancel.cancel();
let (tx, rx) = channel();
spawn_build(dir.path().to_path_buf(), 2, cancel, tx).unwrap();
let summary = loop {
if let Update::Ready(summary) = rx.recv_timeout(Duration::from_secs(30)).unwrap() {
break summary;
}
};
assert!(summary.cancelled);
assert!(summary.files < project.indexed_files);
}
#[test]
fn updater_follows_disk_and_buffers() {
let dir = tempfile::tempdir().unwrap();
let root = dir.path();
write(root, ".gitignore", "*.log\n/ignored/\n");
write(root, "src/A.php", "<?php class Alpha {}");
let (mut index, _, _) = build_index(root, 2);
assert_eq!(index.find_classes("Alpha", 5).len(), 1);
let (tx, rx) = channel();
let updater = Updater::spawn(root.to_path_buf(), CancelToken::new(), tx).unwrap();
write(root, "src/A.php", "<?php class Alpha {} class Beta {}");
updater.send(Job::File {
path: "src/A.php".into(),
text: None,
});
apply_until_quiet(&mut index, &rx);
assert_eq!(index.get("src/A.php").unwrap().symbols.len(), 2);
let buffer = lutra_text::Buffer::new("<?php class Gamma { public function go() {} }");
for _ in 0..5 {
updater.send(Job::File {
path: "src/A.php".into(),
text: Some(buffer.clone()),
});
}
apply_until_quiet(&mut index, &rx);
assert!(index.find_classes("Beta", 5).is_empty());
assert_eq!(index.find_symbols("Gamma::go", 5).len(), 1);
write(root, "src/sub/deep/B.php", "<?php class Delta {}");
write(root, "src/sub/notes.log", "ignored");
updater.send(Job::Dir {
path: "src/sub".into(),
});
apply_until_quiet(&mut index, &rx);
assert!(index.contains("src/sub/deep/B.php"));
assert!(!index.contains("src/sub/notes.log"));
write(root, "ignored/C.php", "<?php class Epsilon {}");
updater.send(Job::File {
path: "ignored/C.php".into(),
text: None,
});
apply_until_quiet(&mut index, &rx);
assert!(!index.contains("ignored/C.php"));
fs::remove_dir_all(root.join("src/sub")).unwrap();
updater.send(Job::Dir {
path: "src/sub".into(),
});
apply_until_quiet(&mut index, &rx);
assert!(!index.contains("src/sub/deep/B.php"));
fs::remove_file(root.join("src/A.php")).unwrap();
updater.send(Job::File {
path: "src/A.php".into(),
text: None,
});
apply_until_quiet(&mut index, &rx);
assert!(!index.contains("src/A.php"));
assert_eq!(index.symbol_count(), 0);
}

View file

@ -0,0 +1,78 @@
//! Index timing on a generated 20,000-file project. Ignored by default; run
//! `cargo test --release -p lutra-index --test perf -- --ignored --nocapture`.
#![allow(
clippy::unwrap_used,
clippy::expect_used,
clippy::indexing_slicing,
clippy::panic
)]
use std::time::Instant;
use lutra_index::{CancelToken, Index, Update, build, default_threads};
#[test]
#[ignore = "measurement; run in release"]
fn index_20k_files() {
let dir = tempfile::tempdir().unwrap();
let project = lutra_testkit::generate_php_project(dir.path(), 20_000);
println!(
"generated: {} indexed files, {} PHP ({} vendor), {:.1} MB PHP, {} classes",
project.indexed_files,
project.php_files,
project.vendor_files,
project.php_bytes as f64 / 1e6,
project.classes
);
let threads = default_threads();
for round in 0..3 {
let index = std::sync::Mutex::new(Index::new());
let mut first_batch = None;
let started = Instant::now();
let first = std::sync::Mutex::new(&mut first_batch);
let summary = build(dir.path(), threads, &CancelToken::new(), &|u| {
if let Update::Files { files, .. } = u {
let mut first = first.lock().unwrap();
if first.is_none() {
**first = Some(started.elapsed());
}
let mut index = index.lock().unwrap();
for f in files {
index.insert(f);
}
}
});
let index = index.into_inner().unwrap();
assert_eq!(index.len(), project.indexed_files);
let time = |f: &dyn Fn() -> usize| {
let mut runs: Vec<f64> = (0..7)
.map(|_| {
let t = Instant::now();
assert!(f() > 0);
t.elapsed().as_secs_f64() * 1e3
})
.collect();
runs.sort_by(f64::total_cmp);
(runs[0], runs[3])
};
let class = time(&|| index.find_classes("Serv12", 50).len());
let symbol = time(&|| index.find_symbols("handle1", 50).len());
let member = time(&|| index.find_symbols("Service12::hand", 50).len());
let file = time(&|| index.find_files("wid12", 50).len());
println!(
"round {round}: {threads} workers, walk {:.0} ms, first batch {:.0} ms, total {:.0} ms, {} files, {} symbols, {} parsed ({:.1} MB)",
summary.walk.as_secs_f64() * 1e3,
first_batch.unwrap().as_secs_f64() * 1e3,
summary.elapsed.as_secs_f64() * 1e3,
summary.files,
summary.symbols,
summary.parsed,
summary.parsed_bytes as f64 / 1e6,
);
println!(
" queries (min / median of 7): goto.class {:.1} / {:.1} ms, goto.symbol {:.1} / {:.1} ms, Class::member {:.1} / {:.1} ms, goto.file {:.1} / {:.1} ms",
class.0, class.1, symbol.0, symbol.1, member.0, member.1, file.0, file.1
);
}
}

View file

@ -0,0 +1,88 @@
//! Property tests: random queries and random file contents never panic, and
//! query results respect the limit and the ranking order.
#![allow(
clippy::unwrap_used,
clippy::expect_used,
clippy::indexing_slicing,
clippy::panic
)]
use lutra_index::{Index, IndexedFile, index_text};
use lutra_syntax::Parser;
use proptest::prelude::*;
/// A small index with project and library files, unicode names included.
fn sample() -> Index {
let mut parser = Parser::new().unwrap();
let mut index = Index::new();
let files = [
(
"src/Models/User.php",
"<?php namespace App\\Models; class User { public function find() {} private $név; const MAX = 1; }",
),
(
"src/Ünïcode/Grüße.php",
"<?php namespace Ü; final class Grüße { public function sägen() {} }",
),
(
"src/helpers.php",
"<?php function format_user() {} const LIMIT = 3;",
),
(
"vendor/acme/User.php",
"<?php namespace Acme; interface User { public function find(); } enum E: int { case A = 1; }",
),
("templates/a😀b.twig", ""),
];
for (path, text) in files {
index.insert(index_text(&mut parser, path.into(), text));
}
index
}
proptest! {
#![proptest_config(ProptestConfig::with_cases(512))]
#[test]
fn random_queries_never_panic(query in "\\PC{0,40}|[\\\\/:$ _a-zA-Z]{0,12}", limit in 0usize..2_000) {
let index = sample();
let files = index.find_files(&query, limit);
let classes = index.find_classes(&query, limit);
let symbols = index.find_symbols(&query, limit);
let cap = limit.min(lutra_index::MAX_LIMIT);
prop_assert!(files.len() <= cap && classes.len() <= cap && symbols.len() <= cap);
for pair in files.windows(2) {
prop_assert!(pair[0].library <= pair[1].library);
if pair[0].library == pair[1].library {
prop_assert!(pair[0].score >= pair[1].score);
}
}
for hit in &files {
let len: usize = hit.path.encode_utf16().count();
for range in &hit.matches {
prop_assert!(range[0] < range[1] && range[1] <= len);
}
prop_assert!(hit.matches.windows(2).all(|w| w[0][1] < w[1][0]));
}
prop_assert!(classes.iter().all(|h| h.symbol.kind.is_class_like()));
prop_assert!(symbols.iter().all(|h| !h.symbol.kind.is_class_like()));
}
#[test]
fn random_sources_never_panic(bytes in proptest::collection::vec(any::<u8>(), 0..2_000)) {
let mut parser = Parser::new().unwrap();
let mut text = String::from("<?php ");
text.push_str(&String::from_utf8_lossy(&bytes));
let file = index_text(&mut parser, "src/x.php".into(), &text);
let mut index = Index::new();
index.insert(file.clone());
for symbol in &file.symbols {
prop_assert!(!symbol.name.is_empty());
let _ = index.find_symbols(&symbol.name, 10);
let _ = index.find_classes(&symbol.name, 10);
}
index.insert(IndexedFile { path: "src/x.php".into(), library: false, symbols: Vec::new() });
prop_assert_eq!(index.symbol_count(), 0);
}
}