M2.5: symbol index and go-to commands

crates/index: walks the project with the search/VFS rules (vendor as
library), parses .php files from tree-sitter outlines on a worker pool
off the engine lock, updates incrementally from disk and buffers through
an updater thread, caps at 200,000 files, and answers fuzzy file, class
and symbol queries (nucleo-matcher, MPL-2.0 as a separate crate).

Engine: index.status, goto.file, goto.class, goto.symbol; events
indexProgress and indexReady drained by the pump; the index starts on
project.open and is cancelled on close. Tests: unit, incremental,
property (random queries and sources), engine and TCP control tests,
release timing on a generated 20k-file project (3.9-4.2 s).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
danny 2026-10-17 12:53:33 +02:00
parent f13a49064d
commit 053441d066
26 changed files with 3455 additions and 9 deletions

667
crates/index/src/query.rs Normal file
View file

@ -0,0 +1,667 @@
//! The index in memory and the fuzzy go-to queries over it.
//!
//! Matching uses `nucleo-matcher` (fzf-style scoring: consecutive
//! characters, word starts and path separators score higher). The query is
//! split at whitespace and every word must match; no other syntax is
//! special. Lower-case query characters match either case, upper-case ones
//! only themselves (smart case). Project files and symbols always rank
//! before library ones; within each group the higher score wins, then the
//! shorter name, then the name in byte order, so results are stable.
use std::cmp::Ordering;
use std::collections::BTreeMap;
use nucleo_matcher::pattern::{AtomKind, CaseMatching, Normalization, Pattern};
use nucleo_matcher::{Config, Matcher, Utf32Str};
use crate::{IndexedFile, MAX_FILES, Symbol};
/// Most results one query returns.
pub const MAX_LIMIT: usize = 1_000;
/// Longest query, in characters; the rest is ignored.
pub const MAX_QUERY_CHARS: usize = 256;
/// Score factor of a match inside the file name over a match in the whole
/// path, so `user` finds `User.php` before `user/Repository.php`.
const NAME_WEIGHT: u32 = 2;
/// Files plus symbols from which a query is scanned on several threads.
const PARALLEL_FROM: usize = 20_000;
/// The files and symbols of a project.
#[derive(Debug, Default)]
pub struct Index {
/// Files by project-relative path.
files: BTreeMap<String, IndexedFile>,
/// Symbols in all files.
symbols: usize,
}
/// A file found by [`Index::find_files`].
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct FileHit {
/// Project-relative path with `/` separators.
pub path: String,
/// Match score; higher is better.
pub score: u32,
/// Matched character ranges in `path`, `[start, end)` in UTF-16 code
/// units, ascending and not overlapping.
pub matches: Vec<[usize; 2]>,
/// Library code (below `vendor/`).
pub library: bool,
}
/// A symbol found by [`Index::find_classes`] or [`Index::find_symbols`].
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct SymbolHit<'a> {
/// The symbol.
pub symbol: &'a Symbol,
/// Project-relative path of its file.
pub path: &'a str,
/// Library code (below `vendor/`).
pub library: bool,
/// Match score; higher is better.
pub score: u32,
}
impl Index {
/// An empty index.
pub fn new() -> Self {
Self::default()
}
/// Number of files.
pub fn len(&self) -> usize {
self.files.len()
}
/// Whether the index holds no files.
pub fn is_empty(&self) -> bool {
self.files.is_empty()
}
/// Number of symbols in all files.
pub fn symbol_count(&self) -> usize {
self.symbols
}
/// The file at the project-relative `path`.
pub fn get(&self, path: &str) -> Option<&IndexedFile> {
self.files.get(path)
}
/// Whether the index holds the file at `path`.
pub fn contains(&self, path: &str) -> bool {
self.files.contains_key(path)
}
/// Adds `file`, replacing the entry with the same path. A new file is
/// refused (returns `false`) when the index already holds
/// [`MAX_FILES`] files.
pub fn insert(&mut self, file: IndexedFile) -> bool {
if self.files.len() >= MAX_FILES && !self.files.contains_key(&file.path) {
return false;
}
self.symbols = self.symbols.saturating_add(file.symbols.len());
if let Some(old) = self.files.insert(file.path.clone(), file) {
self.symbols = self.symbols.saturating_sub(old.symbols.len());
}
true
}
/// Drops the file at `path`, or every file below the directory `path`.
/// Returns the number of files dropped.
pub fn remove(&mut self, path: &str) -> usize {
let path = path.trim_matches('/');
if let Some(old) = self.files.remove(path) {
self.symbols = self.symbols.saturating_sub(old.symbols.len());
return 1;
}
let keys: Vec<String> = if path.is_empty() {
self.files.keys().cloned().collect()
} else {
let start = format!("{path}/");
let end = format!("{path}0");
self.files
.range(start..end)
.map(|(k, _)| k.clone())
.collect()
};
for key in &keys {
if let Some(old) = self.files.remove(key) {
self.symbols = self.symbols.saturating_sub(old.symbols.len());
}
}
keys.len()
}
/// Drops everything.
pub fn clear(&mut self) {
self.files.clear();
self.symbols = 0;
}
/// Files whose project-relative path matches `query`, best first, at
/// most `limit` (capped at [`MAX_LIMIT`]).
///
/// A query without `/` (or `\`, read as `/`) is matched against the
/// file name first, which scores [`NAME_WEIGHT`] times higher, and
/// against the whole path otherwise; a query with `/` against the whole
/// path only. An empty query finds nothing.
pub fn find_files(&self, query: &str, limit: usize) -> Vec<FileHit> {
let query = clamp(query).replace('\\', "/");
let by_path = query.contains('/');
let Some(mut fuzzy) = Fuzzy::new(&query, Config::DEFAULT.match_paths()) else {
return Vec::new();
};
let order = |a: &(&IndexedFile, u32, bool), b: &(&IndexedFile, u32, bool)| {
rank((a.0.library, a.1, &a.0.path), (b.0.library, b.1, &b.0.path))
};
let hits = self.scan(
limit,
|| Fuzzy::new(&query, Config::DEFAULT.match_paths()),
|fuzzy, file, hits| {
let name = file_name(&file.path);
if !by_path && let Some(score) = fuzzy.score(name) {
hits.push((file, score.saturating_mul(NAME_WEIGHT), true));
} else if let Some(score) = fuzzy.score(&file.path) {
hits.push((file, score, false));
}
},
order,
);
hits.into_iter()
.map(|(file, score, on_name)| {
let name = file_name(&file.path);
let (haystack, base) = if on_name {
let prefix = file.path.len().saturating_sub(name.len());
let base = file.path.get(..prefix).map_or(0, utf16_len);
(name, base)
} else {
(file.path.as_str(), 0)
};
FileHit {
path: file.path.clone(),
score,
matches: ranges(haystack, &fuzzy.indices(haystack), base),
library: file.library,
}
})
.collect()
}
/// Classes, interfaces, traits and enums whose name matches `query`,
/// best first, at most `limit` (capped at [`MAX_LIMIT`]). A query with
/// `\` is matched against the fully qualified name instead; a leading
/// `\` is ignored. An empty query finds nothing.
pub fn find_classes(&self, query: &str, limit: usize) -> Vec<SymbolHit<'_>> {
self.find_symbols_where(query, limit, true)
}
/// Functions, methods, properties, constants and enum cases whose name
/// matches `query`, best first, at most `limit` (capped at
/// [`MAX_LIMIT`]). `Class::member` matches the container's short name
/// (or its FQN when the part before `::` has a `\`) and the member name
/// separately; `Class::` lists the members of matching classes. A query
/// with `\` and no `::` is matched against the fully qualified name. An
/// empty query finds nothing.
pub fn find_symbols(&self, query: &str, limit: usize) -> Vec<SymbolHit<'_>> {
self.find_symbols_where(query, limit, false)
}
/// The shared part of [`Index::find_classes`] (`classes`) and
/// [`Index::find_symbols`].
fn find_symbols_where<'a>(
&'a self,
query: &str,
limit: usize,
classes: bool,
) -> Vec<SymbolHit<'a>> {
let query = clamp(query);
let query = query.trim().trim_start_matches('\\');
let config = symbol_config();
let split = if classes {
None
} else {
query.split_once("::")
};
let (owner_query, member_query) = match split {
Some((owner, member)) => (Some(owner.trim_start_matches('\\')), member),
None => (None, query),
};
let owner_by_fqn = owner_query.is_some_and(|q| q.contains('\\'));
let by_fqn = owner_query.is_none() && member_query.contains('\\');
let matchers = || {
let owner = owner_query.and_then(|q| Fuzzy::new(q, config.clone()));
let member = Fuzzy::new(member_query, config.clone());
(owner.is_some() || member.is_some()).then_some((owner, member))
};
if matchers().is_none() {
return Vec::new();
}
let visit = |(owner, member): &mut (Option<Fuzzy>, Option<Fuzzy>),
file: &'a IndexedFile,
hits: &mut Vec<SymbolHit<'a>>| {
for symbol in &file.symbols {
if symbol.kind.is_class_like() != classes {
continue;
}
let mut score = 0u32;
if let Some(owner) = owner.as_mut() {
let Some(container) = symbol.container.as_deref() else {
continue;
};
let haystack = if owner_by_fqn {
container
} else {
container.rsplit('\\').next().unwrap_or(container)
};
let Some(s) = owner.score(haystack) else {
continue;
};
score = s;
}
if let Some(member) = member.as_mut() {
let haystack = if by_fqn { &symbol.fqn } else { &symbol.name };
let Some(s) = member.score(haystack) else {
continue;
};
score = score.saturating_add(s);
}
hits.push(SymbolHit {
symbol,
path: &file.path,
library: file.library,
score,
});
}
};
self.scan(limit, matchers, visit, |a, b| {
rank(
(a.library, a.score, &a.symbol.name),
(b.library, b.score, &b.symbol.name),
)
.then_with(|| a.symbol.fqn.cmp(&b.symbol.fqn))
.then_with(|| a.path.cmp(b.path))
.then_with(|| a.symbol.line.cmp(&b.symbol.line))
})
}
/// Runs `visit` over every file and returns the best `limit` hits in
/// `order`. Large indexes are scanned on [`crate::default_threads`]
/// threads, each with its own matcher state from `make` and its own
/// best hits, merged at the end; small ones on the calling thread.
/// Returns nothing when `make` gives no state (an empty query).
fn scan<'a, S, T: Send>(
&'a self,
limit: usize,
make: impl Fn() -> Option<S> + Sync,
visit: impl Fn(&mut S, &'a IndexedFile, &mut Vec<T>) + Sync,
order: impl Fn(&T, &T) -> Ordering + Sync,
) -> Vec<T> {
let files: Vec<&IndexedFile> = self.files.values().collect();
let work = files.len().saturating_add(self.symbols);
let threads = if work < PARALLEL_FROM {
1
} else {
crate::default_threads()
};
let chunk = files.len().div_ceil(threads).max(1);
let run = |part: &[&'a IndexedFile]| {
let mut hits = Vec::new();
if let Some(mut state) = make() {
for file in part {
visit(&mut state, file, &mut hits);
}
}
best(&mut hits, limit, &order);
hits
};
let mut hits: Vec<T> = if threads <= 1 {
run(&files)
} else {
std::thread::scope(|scope| {
let handles: Vec<_> = files
.chunks(chunk)
.map(|part| scope.spawn(|| run(part)))
.collect();
handles
.into_iter()
.flat_map(|h| h.join().unwrap_or_default())
.collect()
})
};
best(&mut hits, limit, &order);
hits
}
}
/// The ranking order: project before library, higher score first, then
/// shorter, then byte order.
fn rank(a: (bool, u32, &str), b: (bool, u32, &str)) -> Ordering {
a.0.cmp(&b.0)
.then_with(|| b.1.cmp(&a.1))
.then_with(|| a.2.len().cmp(&b.2.len()))
.then_with(|| a.2.cmp(b.2))
}
/// Keeps the `limit` (at most [`MAX_LIMIT`]) first items of `hits` in
/// `order`, sorted.
fn best<T>(hits: &mut Vec<T>, limit: usize, order: impl Fn(&T, &T) -> Ordering) {
let limit = limit.min(MAX_LIMIT);
if limit == 0 {
hits.clear();
return;
}
if hits.len() > limit {
hits.select_nth_unstable_by(limit.saturating_sub(1), &order);
hits.truncate(limit);
}
hits.sort_by(order);
}
/// The matcher configuration for symbol names: the default word
/// boundaries (camel case, `\`, `_`, `:` and other punctuation start
/// words), and matches near the start score a little higher.
fn symbol_config() -> Config {
let mut config = Config::DEFAULT;
config.prefer_prefix = true;
config
}
/// The first [`MAX_QUERY_CHARS`] characters of `query`.
fn clamp(query: &str) -> String {
query.chars().take(MAX_QUERY_CHARS).collect()
}
/// The file name of a project-relative path.
fn file_name(path: &str) -> &str {
path.rsplit('/').next().unwrap_or(path)
}
/// Length of `text` in UTF-16 code units.
fn utf16_len(text: &str) -> usize {
text.chars().map(char::len_utf16).sum()
}
/// Turns sorted character indices into `text` into merged `[start, end)`
/// UTF-16 ranges, shifted by `base`.
fn ranges(text: &str, indices: &[u32], base: usize) -> Vec<[usize; 2]> {
let mut out: Vec<[usize; 2]> = Vec::new();
let mut wanted = indices.iter().copied().peekable();
let mut at = base;
for (i, ch) in text.chars().enumerate() {
let width = ch.len_utf16();
if wanted
.peek()
.is_some_and(|&w| usize::try_from(w).is_ok_and(|w| w == i))
{
wanted.next();
match out.last_mut() {
Some(range) if range[1] == at => range[1] = at.saturating_add(width),
_ => out.push([at, at.saturating_add(width)]),
}
}
at = at.saturating_add(width);
}
out
}
/// A compiled query with the matcher and buffers it needs.
struct Fuzzy {
/// The query words.
pattern: Pattern,
/// The matcher and its scratch memory.
matcher: Matcher,
/// Characters of the current haystack, for non-ASCII text.
chars: Vec<char>,
/// Match indices, reused.
indices: Vec<u32>,
}
impl Fuzzy {
/// The query `query` with `config`; `None` for a query without words.
fn new(query: &str, config: Config) -> Option<Self> {
let pattern = Pattern::new(
query,
CaseMatching::Smart,
Normalization::Smart,
AtomKind::Fuzzy,
);
if pattern.atoms.is_empty() {
return None;
}
Some(Self {
pattern,
matcher: Matcher::new(config),
chars: Vec::new(),
indices: Vec::new(),
})
}
/// The score of `haystack`, `None` if it does not match.
fn score(&mut self, haystack: &str) -> Option<u32> {
let Self {
pattern,
matcher,
chars,
..
} = self;
pattern.score(utf32(haystack, chars), matcher)
}
/// The matched character indices in `haystack`, sorted, without
/// duplicates; empty if it does not match.
fn indices(&mut self, haystack: &str) -> Vec<u32> {
let Self {
pattern,
matcher,
chars,
indices,
} = self;
indices.clear();
if pattern
.indices(utf32(haystack, chars), matcher, indices)
.is_none()
{
return Vec::new();
}
indices.sort_unstable();
indices.dedup();
indices.clone()
}
}
/// `text` as the matcher's string type, one entry per Unicode scalar
/// value, so match indices are character indices (`Utf32Str::new` would
/// merge grapheme clusters).
fn utf32<'a>(text: &'a str, chars: &'a mut Vec<char>) -> Utf32Str<'a> {
if text.is_ascii() {
return Utf32Str::Ascii(text.as_bytes());
}
chars.clear();
chars.extend(text.chars());
Utf32Str::Unicode(chars)
}
#[cfg(test)]
mod tests {
use super::*;
use crate::SymbolKind;
fn file(path: &str, symbols: &[(&str, &str, SymbolKind, Option<&str>)]) -> IndexedFile {
IndexedFile {
path: path.into(),
library: crate::is_library(path),
symbols: symbols
.iter()
.map(|(name, fqn, kind, container)| Symbol {
name: (*name).into(),
fqn: (*fqn).into(),
kind: *kind,
container: container.map(str::to_string),
line: 1,
column: 2,
})
.collect(),
}
}
fn sample() -> Index {
let mut index = Index::new();
index.insert(file(
"src/Models/User.php",
&[
("User", "App\\Models\\User", SymbolKind::Class, None),
(
"find",
"App\\Models\\User::find",
SymbolKind::Method,
Some("App\\Models\\User"),
),
(
"$name",
"App\\Models\\User::$name",
SymbolKind::Property,
Some("App\\Models\\User"),
),
],
));
index.insert(file(
"src/user/Repository.php",
&[(
"Repository",
"App\\User\\Repository",
SymbolKind::Class,
None,
)],
));
index.insert(file(
"vendor/acme/lib/src/User.php",
&[
("User", "Acme\\User", SymbolKind::Class, None),
(
"find",
"Acme\\User::find",
SymbolKind::Method,
Some("Acme\\User"),
),
],
));
index.insert(file("templates/user.html.twig", &[]));
index.insert(file(
"src/helpers.php",
&[("format_user", "format_user", SymbolKind::Function, None)],
));
index
}
#[test]
fn counts_and_removal() {
let mut index = sample();
assert_eq!(index.len(), 5);
assert_eq!(index.symbol_count(), 7);
index.insert(file("src/helpers.php", &[]));
assert_eq!(index.symbol_count(), 6);
assert_eq!(index.remove("src/user"), 1);
assert_eq!(index.remove("src/Models/User.php"), 1);
assert_eq!(index.remove("src/Models"), 0);
assert_eq!(index.symbol_count(), 2);
assert!(index.contains("src/helpers.php"));
assert_eq!(index.remove(""), 3);
assert!(index.is_empty());
assert_eq!(index.symbol_count(), 0);
}
#[test]
fn prefix_removal_respects_segments() {
let mut index = Index::new();
index.insert(file("src/a.php", &[]));
index.insert(file("src2/b.php", &[]));
index.insert(file("src-x/c.php", &[]));
assert_eq!(index.remove("src"), 1);
assert_eq!(index.len(), 2);
}
#[test]
fn files_rank_names_then_paths_and_project_first() {
let index = sample();
let paths: Vec<String> = index
.find_files("user", 10)
.into_iter()
.map(|h| h.path)
.collect();
assert_eq!(
paths,
vec![
"src/Models/User.php",
"templates/user.html.twig",
"src/user/Repository.php",
"vendor/acme/lib/src/User.php",
]
);
let hit = &index.find_files("user", 1)[0];
assert_eq!(hit.matches, vec![[11, 15]]);
let by_path = index.find_files("models/u", 10);
assert_eq!(by_path[0].path, "src/Models/User.php");
assert!(index.find_files(" ", 10).is_empty());
assert!(index.find_files("user", 0).is_empty());
assert!(index.find_files("zzz", 10).is_empty());
}
#[test]
fn classes_and_symbols() {
let index = sample();
let classes: Vec<(&str, bool)> = index
.find_classes("user", 10)
.iter()
.map(|h| (h.symbol.fqn.as_str(), h.library))
.collect();
assert_eq!(
classes,
vec![("App\\Models\\User", false), ("Acme\\User", true)]
);
let fqn = index.find_classes("\\acme\\us", 10);
assert_eq!(fqn.len(), 1);
assert_eq!(fqn[0].symbol.fqn, "Acme\\User");
let finds: Vec<&str> = index
.find_symbols("find", 10)
.iter()
.map(|h| h.symbol.fqn.as_str())
.collect();
assert_eq!(finds, vec!["App\\Models\\User::find", "Acme\\User::find"]);
let members: Vec<&str> = index
.find_symbols("User::", 10)
.iter()
.map(|h| h.symbol.name.as_str())
.collect();
assert_eq!(members, vec!["find", "$name", "find"]);
let scoped = index.find_symbols("acme\\user::f", 10);
assert_eq!(scoped.len(), 1);
assert!(
index
.find_symbols("user", 10)
.iter()
.all(|h| !h.symbol.kind.is_class_like())
);
assert!(index.find_symbols("::", 10).is_empty());
}
#[test]
fn match_ranges_count_utf16() {
assert_eq!(ranges("aéb", &[0, 1, 2], 0), vec![[0, 3]]);
assert_eq!(ranges("a😀b", &[2], 4), vec![[7, 8]]);
assert_eq!(ranges("abc", &[0, 2], 0), vec![[0, 1], [2, 3]]);
}
#[test]
fn caps_the_number_of_files() {
let mut index = Index::new();
for i in 0..MAX_FILES {
index.files.insert(format!("f{i}"), IndexedFile::default());
}
assert!(!index.insert(file("new.php", &[])));
assert!(index.insert(file("f1", &[])));
}
}