//! The index in memory and the fuzzy go-to queries over it. //! //! Matching uses `nucleo-matcher` (fzf-style scoring: consecutive //! characters, word starts and path separators score higher). The query is //! split at whitespace and every word must match; no other syntax is //! special. Lower-case query characters match either case, upper-case ones //! only themselves (smart case). Project files and symbols always rank //! before library ones; within each group the higher score wins, then the //! shorter name, then the name in byte order, so results are stable. use std::cmp::Ordering; use std::collections::BTreeMap; use nucleo_matcher::pattern::{AtomKind, CaseMatching, Normalization, Pattern}; use nucleo_matcher::{Config, Matcher, Utf32Str}; use crate::{IndexedFile, MAX_FILES, Symbol}; /// Most results one query returns. pub const MAX_LIMIT: usize = 1_000; /// Longest query, in characters; the rest is ignored. pub const MAX_QUERY_CHARS: usize = 256; /// Score factor of a match inside the file name over a match in the whole /// path, so `user` finds `User.php` before `user/Repository.php`. const NAME_WEIGHT: u32 = 2; /// Files plus symbols from which a query is scanned on several threads. const PARALLEL_FROM: usize = 20_000; /// The files and symbols of a project. #[derive(Debug, Default)] pub struct Index { /// Files by project-relative path. files: BTreeMap, /// Symbols in all files. symbols: usize, } /// A file found by [`Index::find_files`]. #[derive(Debug, Clone, PartialEq, Eq)] pub struct FileHit { /// Project-relative path with `/` separators. pub path: String, /// Match score; higher is better. pub score: u32, /// Matched character ranges in `path`, `[start, end)` in UTF-16 code /// units, ascending and not overlapping. pub matches: Vec<[usize; 2]>, /// Library code (below `vendor/`). pub library: bool, } /// A symbol found by [`Index::find_classes`] or [`Index::find_symbols`]. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct SymbolHit<'a> { /// The symbol. pub symbol: &'a Symbol, /// Project-relative path of its file. pub path: &'a str, /// Library code (below `vendor/`). pub library: bool, /// Match score; higher is better. pub score: u32, } impl Index { /// An empty index. pub fn new() -> Self { Self::default() } /// Number of files. pub fn len(&self) -> usize { self.files.len() } /// Whether the index holds no files. pub fn is_empty(&self) -> bool { self.files.is_empty() } /// Number of symbols in all files. pub fn symbol_count(&self) -> usize { self.symbols } /// The file at the project-relative `path`. pub fn get(&self, path: &str) -> Option<&IndexedFile> { self.files.get(path) } /// Whether the index holds the file at `path`. pub fn contains(&self, path: &str) -> bool { self.files.contains_key(path) } /// Adds `file`, replacing the entry with the same path. A new file is /// refused (returns `false`) when the index already holds /// [`MAX_FILES`] files. pub fn insert(&mut self, file: IndexedFile) -> bool { if self.files.len() >= MAX_FILES && !self.files.contains_key(&file.path) { return false; } self.symbols = self.symbols.saturating_add(file.symbols.len()); if let Some(old) = self.files.insert(file.path.clone(), file) { self.symbols = self.symbols.saturating_sub(old.symbols.len()); } true } /// Drops the file at `path`, or every file below the directory `path`. /// Returns the number of files dropped. pub fn remove(&mut self, path: &str) -> usize { let path = path.trim_matches('/'); if let Some(old) = self.files.remove(path) { self.symbols = self.symbols.saturating_sub(old.symbols.len()); return 1; } let keys: Vec = if path.is_empty() { self.files.keys().cloned().collect() } else { let start = format!("{path}/"); let end = format!("{path}0"); self.files .range(start..end) .map(|(k, _)| k.clone()) .collect() }; for key in &keys { if let Some(old) = self.files.remove(key) { self.symbols = self.symbols.saturating_sub(old.symbols.len()); } } keys.len() } /// Drops everything. pub fn clear(&mut self) { self.files.clear(); self.symbols = 0; } /// Files whose project-relative path matches `query`, best first, at /// most `limit` (capped at [`MAX_LIMIT`]). /// /// A query without `/` (or `\`, read as `/`) is matched against the /// file name first, which scores [`NAME_WEIGHT`] times higher, and /// against the whole path otherwise; a query with `/` against the whole /// path only. An empty query finds nothing. pub fn find_files(&self, query: &str, limit: usize) -> Vec { let query = clamp(query).replace('\\', "/"); let by_path = query.contains('/'); let Some(mut fuzzy) = Fuzzy::new(&query, Config::DEFAULT.match_paths()) else { return Vec::new(); }; let order = |a: &(&IndexedFile, u32, bool), b: &(&IndexedFile, u32, bool)| { rank((a.0.library, a.1, &a.0.path), (b.0.library, b.1, &b.0.path)) }; let hits = self.scan( limit, || Fuzzy::new(&query, Config::DEFAULT.match_paths()), |fuzzy, file, hits| { let name = file_name(&file.path); if !by_path && let Some(score) = fuzzy.score(name) { hits.push((file, score.saturating_mul(NAME_WEIGHT), true)); } else if let Some(score) = fuzzy.score(&file.path) { hits.push((file, score, false)); } }, order, ); hits.into_iter() .map(|(file, score, on_name)| { let name = file_name(&file.path); let (haystack, base) = if on_name { let prefix = file.path.len().saturating_sub(name.len()); let base = file.path.get(..prefix).map_or(0, utf16_len); (name, base) } else { (file.path.as_str(), 0) }; FileHit { path: file.path.clone(), score, matches: ranges(haystack, &fuzzy.indices(haystack), base), library: file.library, } }) .collect() } /// Classes, interfaces, traits and enums whose name matches `query`, /// best first, at most `limit` (capped at [`MAX_LIMIT`]). A query with /// `\` is matched against the fully qualified name instead; a leading /// `\` is ignored. An empty query finds nothing. pub fn find_classes(&self, query: &str, limit: usize) -> Vec> { self.find_symbols_where(query, limit, true) } /// Functions, methods, properties, constants and enum cases whose name /// matches `query`, best first, at most `limit` (capped at /// [`MAX_LIMIT`]). `Class::member` matches the container's short name /// (or its FQN when the part before `::` has a `\`) and the member name /// separately; `Class::` lists the members of matching classes. A query /// with `\` and no `::` is matched against the fully qualified name. An /// empty query finds nothing. pub fn find_symbols(&self, query: &str, limit: usize) -> Vec> { self.find_symbols_where(query, limit, false) } /// The shared part of [`Index::find_classes`] (`classes`) and /// [`Index::find_symbols`]. fn find_symbols_where<'a>( &'a self, query: &str, limit: usize, classes: bool, ) -> Vec> { let query = clamp(query); let query = query.trim().trim_start_matches('\\'); let config = symbol_config(); let split = if classes { None } else { query.split_once("::") }; let (owner_query, member_query) = match split { Some((owner, member)) => (Some(owner.trim_start_matches('\\')), member), None => (None, query), }; let owner_by_fqn = owner_query.is_some_and(|q| q.contains('\\')); let by_fqn = owner_query.is_none() && member_query.contains('\\'); let matchers = || { let owner = owner_query.and_then(|q| Fuzzy::new(q, config.clone())); let member = Fuzzy::new(member_query, config.clone()); (owner.is_some() || member.is_some()).then_some((owner, member)) }; if matchers().is_none() { return Vec::new(); } let visit = |(owner, member): &mut (Option, Option), file: &'a IndexedFile, hits: &mut Vec>| { for symbol in &file.symbols { if symbol.kind.is_class_like() != classes { continue; } let mut score = 0u32; if let Some(owner) = owner.as_mut() { let Some(container) = symbol.container.as_deref() else { continue; }; let haystack = if owner_by_fqn { container } else { container.rsplit('\\').next().unwrap_or(container) }; let Some(s) = owner.score(haystack) else { continue; }; score = s; } if let Some(member) = member.as_mut() { let haystack = if by_fqn { &symbol.fqn } else { &symbol.name }; let Some(s) = member.score(haystack) else { continue; }; score = score.saturating_add(s); } hits.push(SymbolHit { symbol, path: &file.path, library: file.library, score, }); } }; self.scan(limit, matchers, visit, |a, b| { rank( (a.library, a.score, &a.symbol.name), (b.library, b.score, &b.symbol.name), ) .then_with(|| a.symbol.fqn.cmp(&b.symbol.fqn)) .then_with(|| a.path.cmp(b.path)) .then_with(|| a.symbol.line.cmp(&b.symbol.line)) }) } /// Runs `visit` over every file and returns the best `limit` hits in /// `order`. Large indexes are scanned on [`crate::default_threads`] /// threads, each with its own matcher state from `make` and its own /// best hits, merged at the end; small ones on the calling thread. /// Returns nothing when `make` gives no state (an empty query). fn scan<'a, S, T: Send>( &'a self, limit: usize, make: impl Fn() -> Option + Sync, visit: impl Fn(&mut S, &'a IndexedFile, &mut Vec) + Sync, order: impl Fn(&T, &T) -> Ordering + Sync, ) -> Vec { let files: Vec<&IndexedFile> = self.files.values().collect(); let work = files.len().saturating_add(self.symbols); let threads = if work < PARALLEL_FROM { 1 } else { crate::default_threads() }; let chunk = files.len().div_ceil(threads).max(1); let run = |part: &[&'a IndexedFile]| { let mut hits = Vec::new(); if let Some(mut state) = make() { for file in part { visit(&mut state, file, &mut hits); } } best(&mut hits, limit, &order); hits }; let mut hits: Vec = if threads <= 1 { run(&files) } else { std::thread::scope(|scope| { let handles: Vec<_> = files .chunks(chunk) .map(|part| scope.spawn(|| run(part))) .collect(); handles .into_iter() .flat_map(|h| h.join().unwrap_or_default()) .collect() }) }; best(&mut hits, limit, &order); hits } } /// The ranking order: project before library, higher score first, then /// shorter, then byte order. fn rank(a: (bool, u32, &str), b: (bool, u32, &str)) -> Ordering { a.0.cmp(&b.0) .then_with(|| b.1.cmp(&a.1)) .then_with(|| a.2.len().cmp(&b.2.len())) .then_with(|| a.2.cmp(b.2)) } /// Keeps the `limit` (at most [`MAX_LIMIT`]) first items of `hits` in /// `order`, sorted. fn best(hits: &mut Vec, limit: usize, order: impl Fn(&T, &T) -> Ordering) { let limit = limit.min(MAX_LIMIT); if limit == 0 { hits.clear(); return; } if hits.len() > limit { hits.select_nth_unstable_by(limit.saturating_sub(1), &order); hits.truncate(limit); } hits.sort_by(order); } /// The matcher configuration for symbol names: the default word /// boundaries (camel case, `\`, `_`, `:` and other punctuation start /// words), and matches near the start score a little higher. fn symbol_config() -> Config { let mut config = Config::DEFAULT; config.prefer_prefix = true; config } /// The first [`MAX_QUERY_CHARS`] characters of `query`. fn clamp(query: &str) -> String { query.chars().take(MAX_QUERY_CHARS).collect() } /// The file name of a project-relative path. fn file_name(path: &str) -> &str { path.rsplit('/').next().unwrap_or(path) } /// Length of `text` in UTF-16 code units. fn utf16_len(text: &str) -> usize { text.chars().map(char::len_utf16).sum() } /// Turns sorted character indices into `text` into merged `[start, end)` /// UTF-16 ranges, shifted by `base`. fn ranges(text: &str, indices: &[u32], base: usize) -> Vec<[usize; 2]> { let mut out: Vec<[usize; 2]> = Vec::new(); let mut wanted = indices.iter().copied().peekable(); let mut at = base; for (i, ch) in text.chars().enumerate() { let width = ch.len_utf16(); if wanted .peek() .is_some_and(|&w| usize::try_from(w).is_ok_and(|w| w == i)) { wanted.next(); match out.last_mut() { Some(range) if range[1] == at => range[1] = at.saturating_add(width), _ => out.push([at, at.saturating_add(width)]), } } at = at.saturating_add(width); } out } /// A compiled query with the matcher and buffers it needs. struct Fuzzy { /// The query words. pattern: Pattern, /// The matcher and its scratch memory. matcher: Matcher, /// Characters of the current haystack, for non-ASCII text. chars: Vec, /// Match indices, reused. indices: Vec, } impl Fuzzy { /// The query `query` with `config`; `None` for a query without words. fn new(query: &str, config: Config) -> Option { let pattern = Pattern::new( query, CaseMatching::Smart, Normalization::Smart, AtomKind::Fuzzy, ); if pattern.atoms.is_empty() { return None; } Some(Self { pattern, matcher: Matcher::new(config), chars: Vec::new(), indices: Vec::new(), }) } /// The score of `haystack`, `None` if it does not match. fn score(&mut self, haystack: &str) -> Option { let Self { pattern, matcher, chars, .. } = self; pattern.score(utf32(haystack, chars), matcher) } /// The matched character indices in `haystack`, sorted, without /// duplicates; empty if it does not match. fn indices(&mut self, haystack: &str) -> Vec { let Self { pattern, matcher, chars, indices, } = self; indices.clear(); if pattern .indices(utf32(haystack, chars), matcher, indices) .is_none() { return Vec::new(); } indices.sort_unstable(); indices.dedup(); indices.clone() } } /// `text` as the matcher's string type, one entry per Unicode scalar /// value, so match indices are character indices (`Utf32Str::new` would /// merge grapheme clusters). fn utf32<'a>(text: &'a str, chars: &'a mut Vec) -> Utf32Str<'a> { if text.is_ascii() { return Utf32Str::Ascii(text.as_bytes()); } chars.clear(); chars.extend(text.chars()); Utf32Str::Unicode(chars) } #[cfg(test)] mod tests { use super::*; use crate::SymbolKind; fn file(path: &str, symbols: &[(&str, &str, SymbolKind, Option<&str>)]) -> IndexedFile { IndexedFile { path: path.into(), library: crate::is_library(path), symbols: symbols .iter() .map(|(name, fqn, kind, container)| Symbol { name: (*name).into(), fqn: (*fqn).into(), kind: *kind, container: container.map(str::to_string), line: 1, column: 2, }) .collect(), } } fn sample() -> Index { let mut index = Index::new(); index.insert(file( "src/Models/User.php", &[ ("User", "App\\Models\\User", SymbolKind::Class, None), ( "find", "App\\Models\\User::find", SymbolKind::Method, Some("App\\Models\\User"), ), ( "$name", "App\\Models\\User::$name", SymbolKind::Property, Some("App\\Models\\User"), ), ], )); index.insert(file( "src/user/Repository.php", &[( "Repository", "App\\User\\Repository", SymbolKind::Class, None, )], )); index.insert(file( "vendor/acme/lib/src/User.php", &[ ("User", "Acme\\User", SymbolKind::Class, None), ( "find", "Acme\\User::find", SymbolKind::Method, Some("Acme\\User"), ), ], )); index.insert(file("templates/user.html.twig", &[])); index.insert(file( "src/helpers.php", &[("format_user", "format_user", SymbolKind::Function, None)], )); index } #[test] fn counts_and_removal() { let mut index = sample(); assert_eq!(index.len(), 5); assert_eq!(index.symbol_count(), 7); index.insert(file("src/helpers.php", &[])); assert_eq!(index.symbol_count(), 6); assert_eq!(index.remove("src/user"), 1); assert_eq!(index.remove("src/Models/User.php"), 1); assert_eq!(index.remove("src/Models"), 0); assert_eq!(index.symbol_count(), 2); assert!(index.contains("src/helpers.php")); assert_eq!(index.remove(""), 3); assert!(index.is_empty()); assert_eq!(index.symbol_count(), 0); } #[test] fn prefix_removal_respects_segments() { let mut index = Index::new(); index.insert(file("src/a.php", &[])); index.insert(file("src2/b.php", &[])); index.insert(file("src-x/c.php", &[])); assert_eq!(index.remove("src"), 1); assert_eq!(index.len(), 2); } #[test] fn files_rank_names_then_paths_and_project_first() { let index = sample(); let paths: Vec = index .find_files("user", 10) .into_iter() .map(|h| h.path) .collect(); assert_eq!( paths, vec![ "src/Models/User.php", "templates/user.html.twig", "src/user/Repository.php", "vendor/acme/lib/src/User.php", ] ); let hit = &index.find_files("user", 1)[0]; assert_eq!(hit.matches, vec![[11, 15]]); let by_path = index.find_files("models/u", 10); assert_eq!(by_path[0].path, "src/Models/User.php"); assert!(index.find_files(" ", 10).is_empty()); assert!(index.find_files("user", 0).is_empty()); assert!(index.find_files("zzz", 10).is_empty()); } #[test] fn classes_and_symbols() { let index = sample(); let classes: Vec<(&str, bool)> = index .find_classes("user", 10) .iter() .map(|h| (h.symbol.fqn.as_str(), h.library)) .collect(); assert_eq!( classes, vec![("App\\Models\\User", false), ("Acme\\User", true)] ); let fqn = index.find_classes("\\acme\\us", 10); assert_eq!(fqn.len(), 1); assert_eq!(fqn[0].symbol.fqn, "Acme\\User"); let finds: Vec<&str> = index .find_symbols("find", 10) .iter() .map(|h| h.symbol.fqn.as_str()) .collect(); assert_eq!(finds, vec!["App\\Models\\User::find", "Acme\\User::find"]); let members: Vec<&str> = index .find_symbols("User::", 10) .iter() .map(|h| h.symbol.name.as_str()) .collect(); assert_eq!(members, vec!["find", "$name", "find"]); let scoped = index.find_symbols("acme\\user::f", 10); assert_eq!(scoped.len(), 1); assert!( index .find_symbols("user", 10) .iter() .all(|h| !h.symbol.kind.is_class_like()) ); assert!(index.find_symbols("::", 10).is_empty()); } #[test] fn match_ranges_count_utf16() { assert_eq!(ranges("aéb", &[0, 1, 2], 0), vec![[0, 3]]); assert_eq!(ranges("a😀b", &[2], 4), vec![[7, 8]]); assert_eq!(ranges("abc", &[0, 2], 0), vec![[0, 1], [2, 3]]); } #[test] fn caps_the_number_of_files() { let mut index = Index::new(); for i in 0..MAX_FILES { index.files.insert(format!("f{i}"), IndexedFile::default()); } assert!(!index.insert(file("new.php", &[]))); assert!(index.insert(file("f1", &[]))); } }