crates/index: walks the project with the search/VFS rules (vendor as library), parses .php files from tree-sitter outlines on a worker pool off the engine lock, updates incrementally from disk and buffers through an updater thread, caps at 200,000 files, and answers fuzzy file, class and symbol queries (nucleo-matcher, MPL-2.0 as a separate crate). Engine: index.status, goto.file, goto.class, goto.symbol; events indexProgress and indexReady drained by the pump; the index starts on project.open and is cancelled on close. Tests: unit, incremental, property (random queries and sources), engine and TCP control tests, release timing on a generated 20k-file project (3.9-4.2 s). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
222 lines
7.8 KiB
Rust
222 lines
7.8 KiB
Rust
//! Building the index of a project on disk, incremental updates, and
|
|
//! cancellation.
|
|
|
|
#![allow(
|
|
clippy::unwrap_used,
|
|
clippy::expect_used,
|
|
clippy::indexing_slicing,
|
|
clippy::panic
|
|
)]
|
|
|
|
use std::fs;
|
|
use std::path::Path;
|
|
use std::sync::Mutex;
|
|
use std::sync::mpsc::{Receiver, channel};
|
|
use std::time::Duration;
|
|
|
|
use lutra_index::{CancelToken, Index, Job, Update, Updater, build, spawn_build};
|
|
|
|
/// Builds the index of `root` and applies every update.
|
|
fn build_index(root: &Path, threads: usize) -> (Index, lutra_index::Summary, Vec<(usize, usize)>) {
|
|
let updates = Mutex::new(Vec::new());
|
|
let summary = build(root, threads, &CancelToken::new(), &|u| {
|
|
updates.lock().unwrap().push(u)
|
|
});
|
|
let mut index = Index::new();
|
|
let mut progress = Vec::new();
|
|
for update in updates.into_inner().unwrap() {
|
|
match update {
|
|
Update::Files { files, live } => {
|
|
assert!(!live);
|
|
for file in files {
|
|
assert!(index.insert(file));
|
|
}
|
|
}
|
|
Update::Progress { done, total } => progress.push((done, total)),
|
|
other => panic!("unexpected {other:?}"),
|
|
}
|
|
}
|
|
(index, summary, progress)
|
|
}
|
|
|
|
/// Applies updates from `rx` until one batch arrives and `rx` is quiet.
|
|
fn apply_until_quiet(index: &mut Index, rx: &Receiver<Update>) {
|
|
let first = rx.recv_timeout(Duration::from_secs(10)).expect("an update");
|
|
let mut pending = vec![first];
|
|
while let Ok(next) = rx.recv_timeout(Duration::from_millis(300)) {
|
|
pending.push(next);
|
|
}
|
|
for update in pending {
|
|
match update {
|
|
Update::Files { files, live } => {
|
|
assert!(live);
|
|
for file in files {
|
|
index.insert(file);
|
|
}
|
|
}
|
|
Update::Removed { path } => {
|
|
index.remove(&path);
|
|
}
|
|
other => panic!("unexpected {other:?}"),
|
|
}
|
|
}
|
|
}
|
|
|
|
fn write(root: &Path, rel: &str, text: &str) {
|
|
let path = root.join(rel);
|
|
fs::create_dir_all(path.parent().unwrap()).unwrap();
|
|
fs::write(path, text).unwrap();
|
|
}
|
|
|
|
#[test]
|
|
fn builds_with_the_project_rules() {
|
|
let dir = tempfile::tempdir().unwrap();
|
|
let project = lutra_testkit::generate_php_project(dir.path(), 400);
|
|
write(
|
|
dir.path(),
|
|
"src/Broken.php",
|
|
"<?php\nclass Broken {\n public function ok() {}\n public function bad() { $x = ; }\n",
|
|
);
|
|
write(dir.path(), "src/Latin1.php", "<?php\nclass Caf\u{e9} {}\n");
|
|
fs::write(
|
|
dir.path().join("src/Bad.php"),
|
|
b"<?php\nclass Bad\xff {}\nclass Fine {}\n",
|
|
)
|
|
.unwrap();
|
|
write(dir.path(), ".git/Hidden.php", "<?php class InGit {}");
|
|
for threads in [1, 4] {
|
|
let (index, summary, progress) = build_index(dir.path(), threads);
|
|
assert_eq!(index.len(), project.indexed_files + 3);
|
|
assert_eq!(summary.files, index.len());
|
|
assert_eq!(summary.symbols, index.symbol_count());
|
|
assert_eq!(summary.parsed, project.php_files + 3);
|
|
assert!(!summary.capped && !summary.cancelled);
|
|
let (done, total) = *progress.last().unwrap();
|
|
assert_eq!((done, total), (index.len(), index.len()));
|
|
assert!(progress.windows(2).all(|w| w[0].0 <= w[1].0));
|
|
assert!(index.get("var/cache/Cached0.php").is_none(), "gitignored");
|
|
assert!(
|
|
index.get("node_modules/pkg/Hidden0.php").is_none(),
|
|
"excluded"
|
|
);
|
|
assert!(index.get(".git/Hidden.php").is_none());
|
|
let vendor = index
|
|
.get("vendor/acme/framework/src/Component000/Widget0.php")
|
|
.unwrap();
|
|
assert!(vendor.library, "vendor is walked although gitignored");
|
|
let classes = index.find_classes("Service1", 200);
|
|
let service = classes
|
|
.iter()
|
|
.find(|h| h.symbol.fqn == "App\\Module000\\Service1")
|
|
.unwrap();
|
|
assert_eq!(service.path, "src/Module000/Service1.php");
|
|
assert_eq!((service.symbol.line, service.symbol.column), (12, 6));
|
|
assert_eq!(index.find_classes("Broken", 5)[0].symbol.name, "Broken");
|
|
assert!(!index.find_symbols("Broken::ok", 5).is_empty());
|
|
assert_eq!(index.find_classes("Café", 5)[0].symbol.name, "Café");
|
|
assert!(!index.find_classes("Fine", 5).is_empty());
|
|
let classes_found = index.find_classes("", 10);
|
|
assert!(classes_found.is_empty());
|
|
let all: usize = index
|
|
.find_classes("e", 1_000)
|
|
.iter()
|
|
.filter(|h| h.symbol.kind.is_class_like())
|
|
.count();
|
|
assert!(all > 0);
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn spawned_build_ends_with_ready_and_cancels() {
|
|
let dir = tempfile::tempdir().unwrap();
|
|
let project = lutra_testkit::generate_php_project(dir.path(), 300);
|
|
let (tx, rx) = channel();
|
|
spawn_build(dir.path().to_path_buf(), 2, CancelToken::new(), tx).unwrap();
|
|
let mut files = 0;
|
|
let summary = loop {
|
|
match rx.recv_timeout(Duration::from_secs(30)).unwrap() {
|
|
Update::Files { files: batch, .. } => files += batch.len(),
|
|
Update::Ready(summary) => break summary,
|
|
_ => {}
|
|
}
|
|
};
|
|
assert_eq!(files, project.indexed_files);
|
|
assert_eq!(summary.files, files);
|
|
assert!(rx.recv_timeout(Duration::from_millis(200)).is_err());
|
|
|
|
let cancel = CancelToken::new();
|
|
cancel.cancel();
|
|
let (tx, rx) = channel();
|
|
spawn_build(dir.path().to_path_buf(), 2, cancel, tx).unwrap();
|
|
let summary = loop {
|
|
if let Update::Ready(summary) = rx.recv_timeout(Duration::from_secs(30)).unwrap() {
|
|
break summary;
|
|
}
|
|
};
|
|
assert!(summary.cancelled);
|
|
assert!(summary.files < project.indexed_files);
|
|
}
|
|
|
|
#[test]
|
|
fn updater_follows_disk_and_buffers() {
|
|
let dir = tempfile::tempdir().unwrap();
|
|
let root = dir.path();
|
|
write(root, ".gitignore", "*.log\n/ignored/\n");
|
|
write(root, "src/A.php", "<?php class Alpha {}");
|
|
let (mut index, _, _) = build_index(root, 2);
|
|
assert_eq!(index.find_classes("Alpha", 5).len(), 1);
|
|
let (tx, rx) = channel();
|
|
let updater = Updater::spawn(root.to_path_buf(), CancelToken::new(), tx).unwrap();
|
|
|
|
write(root, "src/A.php", "<?php class Alpha {} class Beta {}");
|
|
updater.send(Job::File {
|
|
path: "src/A.php".into(),
|
|
text: None,
|
|
});
|
|
apply_until_quiet(&mut index, &rx);
|
|
assert_eq!(index.get("src/A.php").unwrap().symbols.len(), 2);
|
|
|
|
let buffer = lutra_text::Buffer::new("<?php class Gamma { public function go() {} }");
|
|
for _ in 0..5 {
|
|
updater.send(Job::File {
|
|
path: "src/A.php".into(),
|
|
text: Some(buffer.clone()),
|
|
});
|
|
}
|
|
apply_until_quiet(&mut index, &rx);
|
|
assert!(index.find_classes("Beta", 5).is_empty());
|
|
assert_eq!(index.find_symbols("Gamma::go", 5).len(), 1);
|
|
|
|
write(root, "src/sub/deep/B.php", "<?php class Delta {}");
|
|
write(root, "src/sub/notes.log", "ignored");
|
|
updater.send(Job::Dir {
|
|
path: "src/sub".into(),
|
|
});
|
|
apply_until_quiet(&mut index, &rx);
|
|
assert!(index.contains("src/sub/deep/B.php"));
|
|
assert!(!index.contains("src/sub/notes.log"));
|
|
|
|
write(root, "ignored/C.php", "<?php class Epsilon {}");
|
|
updater.send(Job::File {
|
|
path: "ignored/C.php".into(),
|
|
text: None,
|
|
});
|
|
apply_until_quiet(&mut index, &rx);
|
|
assert!(!index.contains("ignored/C.php"));
|
|
|
|
fs::remove_dir_all(root.join("src/sub")).unwrap();
|
|
updater.send(Job::Dir {
|
|
path: "src/sub".into(),
|
|
});
|
|
apply_until_quiet(&mut index, &rx);
|
|
assert!(!index.contains("src/sub/deep/B.php"));
|
|
|
|
fs::remove_file(root.join("src/A.php")).unwrap();
|
|
updater.send(Job::File {
|
|
path: "src/A.php".into(),
|
|
text: None,
|
|
});
|
|
apply_until_quiet(&mut index, &rx);
|
|
assert!(!index.contains("src/A.php"));
|
|
assert_eq!(index.symbol_count(), 0);
|
|
}
|