M2.5: symbol index and go-to commands
crates/index: walks the project with the search/VFS rules (vendor as library), parses .php files from tree-sitter outlines on a worker pool off the engine lock, updates incrementally from disk and buffers through an updater thread, caps at 200,000 files, and answers fuzzy file, class and symbol queries (nucleo-matcher, MPL-2.0 as a separate crate). Engine: index.status, goto.file, goto.class, goto.symbol; events indexProgress and indexReady drained by the pump; the index starts on project.open and is cancelled on close. Tests: unit, incremental, property (random queries and sources), engine and TCP control tests, release timing on a generated 20k-file project (3.9-4.2 s). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
f13a49064d
commit
053441d066
26 changed files with 3455 additions and 9 deletions
|
|
@ -12,6 +12,15 @@ publish.workspace = true
|
|||
layer = 2
|
||||
|
||||
[dependencies]
|
||||
ignore.workspace = true
|
||||
lutra-syntax.workspace = true
|
||||
lutra-text.workspace = true
|
||||
nucleo-matcher = { version = "0.3.1", default-features = false, features = ["unicode-normalization", "unicode-casefold"] }
|
||||
|
||||
[dev-dependencies]
|
||||
lutra-testkit.workspace = true
|
||||
proptest.workspace = true
|
||||
tempfile.workspace = true
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
|
|
|
|||
257
crates/index/src/build.rs
Normal file
257
crates/index/src/build.rs
Normal file
|
|
@ -0,0 +1,257 @@
|
|||
//! Building the index of a whole project in parallel.
|
||||
|
||||
use std::io::Read;
|
||||
use std::panic::{AssertUnwindSafe, catch_unwind};
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering};
|
||||
use std::sync::mpsc::Sender;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use lutra_syntax::Parser;
|
||||
|
||||
use crate::symbol::{is_library, is_php};
|
||||
use crate::walk::{Found, walk};
|
||||
use crate::{
|
||||
BATCH_FILES, CancelToken, IndexedFile, MAX_FILES, MAX_PARSE_BYTES, Summary, Update, index_text,
|
||||
};
|
||||
|
||||
/// Longest a worker keeps indexed files before sending them.
|
||||
const FLUSH_AFTER: Duration = Duration::from_millis(100);
|
||||
|
||||
/// Files a worker collects before sending them.
|
||||
const WORKER_BATCH: usize = 64;
|
||||
|
||||
/// Walks the project at `root` and indexes it, reporting [`Update::Files`]
|
||||
/// batches and [`Update::Progress`] through `send` while it runs (from the
|
||||
/// calling thread and the workers). Blocks until every file is indexed or
|
||||
/// `cancel` fires. Does not send [`Update::Ready`]; the caller turns the
|
||||
/// returned summary into it.
|
||||
///
|
||||
/// Files without symbols (not PHP, or larger than [`MAX_PARSE_BYTES`]) are
|
||||
/// reported right after the walk. PHP files are then parsed on `threads`
|
||||
/// workers, project files before library files and larger files first
|
||||
/// within each group, so the longest parses start early. A file that
|
||||
/// cannot be read is listed without symbols.
|
||||
pub fn build(
|
||||
root: &Path,
|
||||
threads: usize,
|
||||
cancel: &CancelToken,
|
||||
send: &(dyn Fn(Update) + Sync),
|
||||
) -> Summary {
|
||||
let started = Instant::now();
|
||||
let walked = walk(root, None, threads, MAX_FILES, cancel);
|
||||
let walk_time = started.elapsed();
|
||||
let total = walked.files.len();
|
||||
let (mut php, rest): (Vec<Found>, Vec<Found>) = walked
|
||||
.files
|
||||
.into_iter()
|
||||
.partition(|f| is_php(&f.path) && f.size <= MAX_PARSE_BYTES);
|
||||
let mut done = 0usize;
|
||||
let mut batch = Vec::with_capacity(BATCH_FILES);
|
||||
for found in rest {
|
||||
batch.push(IndexedFile {
|
||||
library: is_library(&found.path),
|
||||
path: found.path,
|
||||
symbols: Vec::new(),
|
||||
});
|
||||
if batch.len() >= BATCH_FILES {
|
||||
done = done.saturating_add(batch.len());
|
||||
send(Update::Files {
|
||||
files: std::mem::take(&mut batch),
|
||||
live: false,
|
||||
});
|
||||
}
|
||||
}
|
||||
done = done.saturating_add(batch.len());
|
||||
if !batch.is_empty() {
|
||||
send(Update::Files {
|
||||
files: batch,
|
||||
live: false,
|
||||
});
|
||||
}
|
||||
send(Update::Progress { done, total });
|
||||
php.sort_by(|a, b| {
|
||||
is_library(&a.path)
|
||||
.cmp(&is_library(&b.path))
|
||||
.then(b.size.cmp(&a.size))
|
||||
});
|
||||
let shared = Shared {
|
||||
files: &php,
|
||||
next: AtomicUsize::new(0),
|
||||
done: AtomicUsize::new(done),
|
||||
total,
|
||||
symbols: AtomicUsize::new(0),
|
||||
parsed: AtomicUsize::new(0),
|
||||
parsed_bytes: AtomicU64::new(0),
|
||||
cancel,
|
||||
send,
|
||||
};
|
||||
let workers = threads.clamp(1, php.len().max(1));
|
||||
std::thread::scope(|scope| {
|
||||
for _ in 0..workers {
|
||||
scope.spawn(|| shared.work());
|
||||
}
|
||||
});
|
||||
Summary {
|
||||
files: shared.done.load(Ordering::Relaxed),
|
||||
symbols: shared.symbols.load(Ordering::Relaxed),
|
||||
parsed: shared.parsed.load(Ordering::Relaxed),
|
||||
parsed_bytes: shared.parsed_bytes.load(Ordering::Relaxed),
|
||||
capped: walked.capped,
|
||||
cancelled: cancel.is_cancelled(),
|
||||
walk: walk_time,
|
||||
elapsed: started.elapsed(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Runs [`build`] on a thread of its own and sends its updates, then
|
||||
/// [`Update::Ready`], over `updates`. Updates sent after the receiver is
|
||||
/// gone are dropped; the caller cancels the build when it stops listening.
|
||||
///
|
||||
/// # Errors
|
||||
/// The thread could not be started.
|
||||
pub fn spawn_build(
|
||||
root: PathBuf,
|
||||
threads: usize,
|
||||
cancel: CancelToken,
|
||||
updates: Sender<Update>,
|
||||
) -> std::io::Result<()> {
|
||||
std::thread::Builder::new()
|
||||
.name("lutra-index".into())
|
||||
.spawn(move || {
|
||||
let send = |update| {
|
||||
let _ = updates.send(update);
|
||||
};
|
||||
let summary = build(&root, threads, &cancel, &send);
|
||||
send(Update::Ready(summary));
|
||||
})
|
||||
.map(|_| ())
|
||||
}
|
||||
|
||||
/// State the workers of one build share.
|
||||
struct Shared<'a> {
|
||||
/// PHP files to parse, in order.
|
||||
files: &'a [Found],
|
||||
/// Index of the next file to take.
|
||||
next: AtomicUsize,
|
||||
/// Files indexed and sent.
|
||||
done: AtomicUsize,
|
||||
/// Files found by the walk.
|
||||
total: usize,
|
||||
/// Symbols found.
|
||||
symbols: AtomicUsize,
|
||||
/// Files parsed.
|
||||
parsed: AtomicUsize,
|
||||
/// Bytes parsed.
|
||||
parsed_bytes: AtomicU64,
|
||||
/// Stops the workers.
|
||||
cancel: &'a CancelToken,
|
||||
/// Receives the batches.
|
||||
send: &'a (dyn Fn(Update) + Sync),
|
||||
}
|
||||
|
||||
impl Shared<'_> {
|
||||
/// One worker: takes files until none are left or the build is
|
||||
/// cancelled, and sends them in batches.
|
||||
fn work(&self) {
|
||||
let mut parser = Parser::new().ok();
|
||||
let mut batch = Vec::with_capacity(WORKER_BATCH);
|
||||
let mut last = Instant::now();
|
||||
while !self.cancel.is_cancelled() {
|
||||
let i = self.next.fetch_add(1, Ordering::Relaxed);
|
||||
let Some(found) = self.files.get(i) else {
|
||||
break;
|
||||
};
|
||||
let (file, bytes) = index_file(&mut parser, found);
|
||||
if let Some(bytes) = bytes {
|
||||
self.parsed.fetch_add(1, Ordering::Relaxed);
|
||||
self.parsed_bytes.fetch_add(bytes, Ordering::Relaxed);
|
||||
}
|
||||
self.symbols
|
||||
.fetch_add(file.symbols.len(), Ordering::Relaxed);
|
||||
batch.push(file);
|
||||
if batch.len() >= WORKER_BATCH || last.elapsed() >= FLUSH_AFTER {
|
||||
self.flush(&mut batch);
|
||||
last = Instant::now();
|
||||
}
|
||||
}
|
||||
self.flush(&mut batch);
|
||||
}
|
||||
|
||||
/// Sends `batch` and the progress after it.
|
||||
fn flush(&self, batch: &mut Vec<IndexedFile>) {
|
||||
if batch.is_empty() {
|
||||
return;
|
||||
}
|
||||
let count = batch.len();
|
||||
(self.send)(Update::Files {
|
||||
files: std::mem::take(batch),
|
||||
live: false,
|
||||
});
|
||||
let done = self
|
||||
.done
|
||||
.fetch_add(count, Ordering::Relaxed)
|
||||
.saturating_add(count);
|
||||
(self.send)(Update::Progress {
|
||||
done,
|
||||
total: self.total,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
/// Reads and indexes one file. Returns the file and the bytes parsed,
|
||||
/// `None` when it was listed without symbols: it is not a PHP file, could
|
||||
/// not be read, grew past [`MAX_PARSE_BYTES`], no parser is available, or
|
||||
/// parsing panicked.
|
||||
pub(crate) fn index_file(parser: &mut Option<Parser>, found: &Found) -> (IndexedFile, Option<u64>) {
|
||||
let empty = || IndexedFile {
|
||||
path: found.path.clone(),
|
||||
library: is_library(&found.path),
|
||||
symbols: Vec::new(),
|
||||
};
|
||||
if !is_php(&found.path) {
|
||||
return (empty(), None);
|
||||
}
|
||||
let Some(text) = read_source(&found.abs) else {
|
||||
return (empty(), None);
|
||||
};
|
||||
match guarded_index(parser, found.path.clone(), &text) {
|
||||
Some(file) => (file, Some(text.len() as u64)),
|
||||
None => (empty(), None),
|
||||
}
|
||||
}
|
||||
|
||||
/// [`index_text`] with a guard: `None` when no parser is available or
|
||||
/// parsing panicked (a bug in the parser; the parser is replaced so the
|
||||
/// next file gets a fresh one, and the index goes on without this file's
|
||||
/// symbols).
|
||||
pub(crate) fn guarded_index(
|
||||
parser: &mut Option<Parser>,
|
||||
path: String,
|
||||
text: &str,
|
||||
) -> Option<IndexedFile> {
|
||||
let p = parser.as_mut()?;
|
||||
match catch_unwind(AssertUnwindSafe(|| index_text(p, path, text))) {
|
||||
Ok(file) => Some(file),
|
||||
Err(_) => {
|
||||
*parser = Parser::new().ok();
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// The text of the source file at `path`: at most [`MAX_PARSE_BYTES`],
|
||||
/// without a UTF-8 byte order mark, invalid UTF-8 replaced. `None` if it
|
||||
/// cannot be read or is larger.
|
||||
pub(crate) fn read_source(path: &Path) -> Option<String> {
|
||||
let file = std::fs::File::open(path).ok()?;
|
||||
let mut bytes = Vec::new();
|
||||
file.take(MAX_PARSE_BYTES.saturating_add(1))
|
||||
.read_to_end(&mut bytes)
|
||||
.ok()?;
|
||||
if bytes.len() as u64 > MAX_PARSE_BYTES {
|
||||
return None;
|
||||
}
|
||||
let body = bytes.strip_prefix(b"\xEF\xBB\xBF").unwrap_or(&bytes);
|
||||
Some(String::from_utf8_lossy(body).into_owned())
|
||||
}
|
||||
|
|
@ -1,4 +1,145 @@
|
|||
//! Symbol index for Lutra (layer L2).
|
||||
//!
|
||||
//! Stores declarations and usages for go-to and find-usages, starting with
|
||||
//! symbols extracted from tree-sitter trees.
|
||||
//! Lists every file of a project and the declarations of its PHP files
|
||||
//! (classes, interfaces, traits, enums, functions, methods, properties,
|
||||
//! constants, enum cases), extracted from tree-sitter outlines
|
||||
//! ([`lutra_syntax::outline`]), and answers fuzzy go-to queries over them.
|
||||
//!
|
||||
//! # Which files
|
||||
//! The walk follows the rules of `lutra-vfs` and `lutra-search`:
|
||||
//! `.gitignore`, `.ignore`, `.git/info/exclude` and the global git excludes
|
||||
//! apply without a git repository; `.git` and every `node_modules` directory
|
||||
//! are skipped; the root's `vendor/` is walked even when an ignore file
|
||||
//! lists it and its files are marked as library code; dot files are listed;
|
||||
//! symbolic links are not followed. Every file is listed for
|
||||
//! [`Index::find_files`]; only `.php` files are parsed, and only those up to
|
||||
//! [`MAX_PARSE_BYTES`]. At most [`MAX_FILES`] files are indexed.
|
||||
//!
|
||||
//! # Building
|
||||
//! [`build`] walks the project, then parses the PHP files on a pool of
|
||||
//! worker threads (one parser each, one file in memory each), project files
|
||||
//! before library files and larger files first within each group. It
|
||||
//! reports [`Update`]s through a callback: batches of indexed files,
|
||||
//! progress, and a final [`Summary`]. [`spawn_build`] runs it on a thread of
|
||||
//! its own and sends the updates over a channel. [`Updater`] re-indexes
|
||||
//! single files and directories later, from disk or from an editor buffer.
|
||||
//! A [`CancelToken`] stops both between files.
|
||||
//!
|
||||
//! # Robustness
|
||||
//! Source files are untrusted: reads are capped, invalid UTF-8 is replaced,
|
||||
//! broken PHP yields the declarations that could be recognised, symbols per
|
||||
//! file are capped ([`MAX_SYMBOLS_PER_FILE`]), and a panic while parsing one
|
||||
//! file (a bug) loses that file's symbols, not the index.
|
||||
|
||||
mod build;
|
||||
mod query;
|
||||
mod symbol;
|
||||
mod updater;
|
||||
mod walk;
|
||||
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::time::Duration;
|
||||
|
||||
pub use build::{build, spawn_build};
|
||||
pub use query::{FileHit, Index, MAX_LIMIT, MAX_QUERY_CHARS, SymbolHit};
|
||||
pub use symbol::{IndexedFile, Symbol, SymbolKind, index_text, is_library, symbols_from_outline};
|
||||
pub use updater::{Job, Updater};
|
||||
|
||||
/// Most files one index holds.
|
||||
pub const MAX_FILES: usize = 200_000;
|
||||
|
||||
/// Largest PHP file parsed, in bytes (2 MiB). Larger files are listed for
|
||||
/// [`Index::find_files`] without symbols: tree-sitter-php parses about
|
||||
/// 2–3 MB/s, so one such file would hold a worker for a second or more, and
|
||||
/// files this size are generated code rather than declarations to navigate.
|
||||
pub const MAX_PARSE_BYTES: u64 = 2 * 1024 * 1024;
|
||||
|
||||
/// Most symbols kept from one file; the rest are dropped.
|
||||
pub const MAX_SYMBOLS_PER_FILE: usize = 20_000;
|
||||
|
||||
/// Most worker threads of one build.
|
||||
pub const MAX_THREADS: usize = 32;
|
||||
|
||||
/// Files per [`Update::Files`] batch of a build.
|
||||
pub const BATCH_FILES: usize = 256;
|
||||
|
||||
/// What a build or an [`Updater`] reports.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub enum Update {
|
||||
/// Indexed files that replace any earlier entry with the same path.
|
||||
Files {
|
||||
/// The files.
|
||||
files: Vec<IndexedFile>,
|
||||
/// From an [`Updater`] (a change after the build started), so newer
|
||||
/// than anything the build reports for the same path.
|
||||
live: bool,
|
||||
},
|
||||
/// A file or directory is gone: drop `path` and everything below it.
|
||||
Removed {
|
||||
/// Project-relative path with `/` separators.
|
||||
path: String,
|
||||
},
|
||||
/// Build progress: `done` of `total` files are indexed.
|
||||
Progress {
|
||||
/// Files indexed so far.
|
||||
done: usize,
|
||||
/// Files found by the walk.
|
||||
total: usize,
|
||||
},
|
||||
/// The build ended; nothing from it follows.
|
||||
Ready(Summary),
|
||||
}
|
||||
|
||||
/// How a build ended.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
|
||||
pub struct Summary {
|
||||
/// Files indexed.
|
||||
pub files: usize,
|
||||
/// Symbols found in them.
|
||||
pub symbols: usize,
|
||||
/// PHP files parsed.
|
||||
pub parsed: usize,
|
||||
/// Bytes of PHP parsed.
|
||||
pub parsed_bytes: u64,
|
||||
/// The walk found more than [`MAX_FILES`] files; the rest are left out.
|
||||
pub capped: bool,
|
||||
/// The build was cancelled before it finished.
|
||||
pub cancelled: bool,
|
||||
/// Wall time of the walk alone.
|
||||
pub walk: Duration,
|
||||
/// Wall time from start to end.
|
||||
pub elapsed: Duration,
|
||||
}
|
||||
|
||||
/// Stops a running build or updater. Clones share the flag.
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub struct CancelToken(Arc<AtomicBool>);
|
||||
|
||||
impl CancelToken {
|
||||
/// A token that is not cancelled.
|
||||
pub fn new() -> Self {
|
||||
Self::default()
|
||||
}
|
||||
|
||||
/// Asks the work to stop at the next file.
|
||||
pub fn cancel(&self) {
|
||||
self.0.store(true, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
/// Whether [`CancelToken::cancel`] was called.
|
||||
pub fn is_cancelled(&self) -> bool {
|
||||
self.0.load(Ordering::Relaxed)
|
||||
}
|
||||
}
|
||||
|
||||
/// Worker threads for a build on this machine: one less than the cores,
|
||||
/// so the UI and the engine keep a core, at least one and at most
|
||||
/// [`MAX_THREADS`].
|
||||
pub fn default_threads() -> usize {
|
||||
std::thread::available_parallelism()
|
||||
.map(|n| n.get())
|
||||
.unwrap_or(2)
|
||||
.saturating_sub(1)
|
||||
.clamp(1, MAX_THREADS)
|
||||
}
|
||||
|
|
|
|||
667
crates/index/src/query.rs
Normal file
667
crates/index/src/query.rs
Normal file
|
|
@ -0,0 +1,667 @@
|
|||
//! The index in memory and the fuzzy go-to queries over it.
|
||||
//!
|
||||
//! Matching uses `nucleo-matcher` (fzf-style scoring: consecutive
|
||||
//! characters, word starts and path separators score higher). The query is
|
||||
//! split at whitespace and every word must match; no other syntax is
|
||||
//! special. Lower-case query characters match either case, upper-case ones
|
||||
//! only themselves (smart case). Project files and symbols always rank
|
||||
//! before library ones; within each group the higher score wins, then the
|
||||
//! shorter name, then the name in byte order, so results are stable.
|
||||
|
||||
use std::cmp::Ordering;
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
use nucleo_matcher::pattern::{AtomKind, CaseMatching, Normalization, Pattern};
|
||||
use nucleo_matcher::{Config, Matcher, Utf32Str};
|
||||
|
||||
use crate::{IndexedFile, MAX_FILES, Symbol};
|
||||
|
||||
/// Most results one query returns.
|
||||
pub const MAX_LIMIT: usize = 1_000;
|
||||
|
||||
/// Longest query, in characters; the rest is ignored.
|
||||
pub const MAX_QUERY_CHARS: usize = 256;
|
||||
|
||||
/// Score factor of a match inside the file name over a match in the whole
|
||||
/// path, so `user` finds `User.php` before `user/Repository.php`.
|
||||
const NAME_WEIGHT: u32 = 2;
|
||||
|
||||
/// Files plus symbols from which a query is scanned on several threads.
|
||||
const PARALLEL_FROM: usize = 20_000;
|
||||
|
||||
/// The files and symbols of a project.
|
||||
#[derive(Debug, Default)]
|
||||
pub struct Index {
|
||||
/// Files by project-relative path.
|
||||
files: BTreeMap<String, IndexedFile>,
|
||||
/// Symbols in all files.
|
||||
symbols: usize,
|
||||
}
|
||||
|
||||
/// A file found by [`Index::find_files`].
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct FileHit {
|
||||
/// Project-relative path with `/` separators.
|
||||
pub path: String,
|
||||
/// Match score; higher is better.
|
||||
pub score: u32,
|
||||
/// Matched character ranges in `path`, `[start, end)` in UTF-16 code
|
||||
/// units, ascending and not overlapping.
|
||||
pub matches: Vec<[usize; 2]>,
|
||||
/// Library code (below `vendor/`).
|
||||
pub library: bool,
|
||||
}
|
||||
|
||||
/// A symbol found by [`Index::find_classes`] or [`Index::find_symbols`].
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub struct SymbolHit<'a> {
|
||||
/// The symbol.
|
||||
pub symbol: &'a Symbol,
|
||||
/// Project-relative path of its file.
|
||||
pub path: &'a str,
|
||||
/// Library code (below `vendor/`).
|
||||
pub library: bool,
|
||||
/// Match score; higher is better.
|
||||
pub score: u32,
|
||||
}
|
||||
|
||||
impl Index {
|
||||
/// An empty index.
|
||||
pub fn new() -> Self {
|
||||
Self::default()
|
||||
}
|
||||
|
||||
/// Number of files.
|
||||
pub fn len(&self) -> usize {
|
||||
self.files.len()
|
||||
}
|
||||
|
||||
/// Whether the index holds no files.
|
||||
pub fn is_empty(&self) -> bool {
|
||||
self.files.is_empty()
|
||||
}
|
||||
|
||||
/// Number of symbols in all files.
|
||||
pub fn symbol_count(&self) -> usize {
|
||||
self.symbols
|
||||
}
|
||||
|
||||
/// The file at the project-relative `path`.
|
||||
pub fn get(&self, path: &str) -> Option<&IndexedFile> {
|
||||
self.files.get(path)
|
||||
}
|
||||
|
||||
/// Whether the index holds the file at `path`.
|
||||
pub fn contains(&self, path: &str) -> bool {
|
||||
self.files.contains_key(path)
|
||||
}
|
||||
|
||||
/// Adds `file`, replacing the entry with the same path. A new file is
|
||||
/// refused (returns `false`) when the index already holds
|
||||
/// [`MAX_FILES`] files.
|
||||
pub fn insert(&mut self, file: IndexedFile) -> bool {
|
||||
if self.files.len() >= MAX_FILES && !self.files.contains_key(&file.path) {
|
||||
return false;
|
||||
}
|
||||
self.symbols = self.symbols.saturating_add(file.symbols.len());
|
||||
if let Some(old) = self.files.insert(file.path.clone(), file) {
|
||||
self.symbols = self.symbols.saturating_sub(old.symbols.len());
|
||||
}
|
||||
true
|
||||
}
|
||||
|
||||
/// Drops the file at `path`, or every file below the directory `path`.
|
||||
/// Returns the number of files dropped.
|
||||
pub fn remove(&mut self, path: &str) -> usize {
|
||||
let path = path.trim_matches('/');
|
||||
if let Some(old) = self.files.remove(path) {
|
||||
self.symbols = self.symbols.saturating_sub(old.symbols.len());
|
||||
return 1;
|
||||
}
|
||||
let keys: Vec<String> = if path.is_empty() {
|
||||
self.files.keys().cloned().collect()
|
||||
} else {
|
||||
let start = format!("{path}/");
|
||||
let end = format!("{path}0");
|
||||
self.files
|
||||
.range(start..end)
|
||||
.map(|(k, _)| k.clone())
|
||||
.collect()
|
||||
};
|
||||
for key in &keys {
|
||||
if let Some(old) = self.files.remove(key) {
|
||||
self.symbols = self.symbols.saturating_sub(old.symbols.len());
|
||||
}
|
||||
}
|
||||
keys.len()
|
||||
}
|
||||
|
||||
/// Drops everything.
|
||||
pub fn clear(&mut self) {
|
||||
self.files.clear();
|
||||
self.symbols = 0;
|
||||
}
|
||||
|
||||
/// Files whose project-relative path matches `query`, best first, at
|
||||
/// most `limit` (capped at [`MAX_LIMIT`]).
|
||||
///
|
||||
/// A query without `/` (or `\`, read as `/`) is matched against the
|
||||
/// file name first, which scores [`NAME_WEIGHT`] times higher, and
|
||||
/// against the whole path otherwise; a query with `/` against the whole
|
||||
/// path only. An empty query finds nothing.
|
||||
pub fn find_files(&self, query: &str, limit: usize) -> Vec<FileHit> {
|
||||
let query = clamp(query).replace('\\', "/");
|
||||
let by_path = query.contains('/');
|
||||
let Some(mut fuzzy) = Fuzzy::new(&query, Config::DEFAULT.match_paths()) else {
|
||||
return Vec::new();
|
||||
};
|
||||
let order = |a: &(&IndexedFile, u32, bool), b: &(&IndexedFile, u32, bool)| {
|
||||
rank((a.0.library, a.1, &a.0.path), (b.0.library, b.1, &b.0.path))
|
||||
};
|
||||
let hits = self.scan(
|
||||
limit,
|
||||
|| Fuzzy::new(&query, Config::DEFAULT.match_paths()),
|
||||
|fuzzy, file, hits| {
|
||||
let name = file_name(&file.path);
|
||||
if !by_path && let Some(score) = fuzzy.score(name) {
|
||||
hits.push((file, score.saturating_mul(NAME_WEIGHT), true));
|
||||
} else if let Some(score) = fuzzy.score(&file.path) {
|
||||
hits.push((file, score, false));
|
||||
}
|
||||
},
|
||||
order,
|
||||
);
|
||||
hits.into_iter()
|
||||
.map(|(file, score, on_name)| {
|
||||
let name = file_name(&file.path);
|
||||
let (haystack, base) = if on_name {
|
||||
let prefix = file.path.len().saturating_sub(name.len());
|
||||
let base = file.path.get(..prefix).map_or(0, utf16_len);
|
||||
(name, base)
|
||||
} else {
|
||||
(file.path.as_str(), 0)
|
||||
};
|
||||
FileHit {
|
||||
path: file.path.clone(),
|
||||
score,
|
||||
matches: ranges(haystack, &fuzzy.indices(haystack), base),
|
||||
library: file.library,
|
||||
}
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Classes, interfaces, traits and enums whose name matches `query`,
|
||||
/// best first, at most `limit` (capped at [`MAX_LIMIT`]). A query with
|
||||
/// `\` is matched against the fully qualified name instead; a leading
|
||||
/// `\` is ignored. An empty query finds nothing.
|
||||
pub fn find_classes(&self, query: &str, limit: usize) -> Vec<SymbolHit<'_>> {
|
||||
self.find_symbols_where(query, limit, true)
|
||||
}
|
||||
|
||||
/// Functions, methods, properties, constants and enum cases whose name
|
||||
/// matches `query`, best first, at most `limit` (capped at
|
||||
/// [`MAX_LIMIT`]). `Class::member` matches the container's short name
|
||||
/// (or its FQN when the part before `::` has a `\`) and the member name
|
||||
/// separately; `Class::` lists the members of matching classes. A query
|
||||
/// with `\` and no `::` is matched against the fully qualified name. An
|
||||
/// empty query finds nothing.
|
||||
pub fn find_symbols(&self, query: &str, limit: usize) -> Vec<SymbolHit<'_>> {
|
||||
self.find_symbols_where(query, limit, false)
|
||||
}
|
||||
|
||||
/// The shared part of [`Index::find_classes`] (`classes`) and
|
||||
/// [`Index::find_symbols`].
|
||||
fn find_symbols_where<'a>(
|
||||
&'a self,
|
||||
query: &str,
|
||||
limit: usize,
|
||||
classes: bool,
|
||||
) -> Vec<SymbolHit<'a>> {
|
||||
let query = clamp(query);
|
||||
let query = query.trim().trim_start_matches('\\');
|
||||
let config = symbol_config();
|
||||
let split = if classes {
|
||||
None
|
||||
} else {
|
||||
query.split_once("::")
|
||||
};
|
||||
let (owner_query, member_query) = match split {
|
||||
Some((owner, member)) => (Some(owner.trim_start_matches('\\')), member),
|
||||
None => (None, query),
|
||||
};
|
||||
let owner_by_fqn = owner_query.is_some_and(|q| q.contains('\\'));
|
||||
let by_fqn = owner_query.is_none() && member_query.contains('\\');
|
||||
let matchers = || {
|
||||
let owner = owner_query.and_then(|q| Fuzzy::new(q, config.clone()));
|
||||
let member = Fuzzy::new(member_query, config.clone());
|
||||
(owner.is_some() || member.is_some()).then_some((owner, member))
|
||||
};
|
||||
if matchers().is_none() {
|
||||
return Vec::new();
|
||||
}
|
||||
let visit = |(owner, member): &mut (Option<Fuzzy>, Option<Fuzzy>),
|
||||
file: &'a IndexedFile,
|
||||
hits: &mut Vec<SymbolHit<'a>>| {
|
||||
for symbol in &file.symbols {
|
||||
if symbol.kind.is_class_like() != classes {
|
||||
continue;
|
||||
}
|
||||
let mut score = 0u32;
|
||||
if let Some(owner) = owner.as_mut() {
|
||||
let Some(container) = symbol.container.as_deref() else {
|
||||
continue;
|
||||
};
|
||||
let haystack = if owner_by_fqn {
|
||||
container
|
||||
} else {
|
||||
container.rsplit('\\').next().unwrap_or(container)
|
||||
};
|
||||
let Some(s) = owner.score(haystack) else {
|
||||
continue;
|
||||
};
|
||||
score = s;
|
||||
}
|
||||
if let Some(member) = member.as_mut() {
|
||||
let haystack = if by_fqn { &symbol.fqn } else { &symbol.name };
|
||||
let Some(s) = member.score(haystack) else {
|
||||
continue;
|
||||
};
|
||||
score = score.saturating_add(s);
|
||||
}
|
||||
hits.push(SymbolHit {
|
||||
symbol,
|
||||
path: &file.path,
|
||||
library: file.library,
|
||||
score,
|
||||
});
|
||||
}
|
||||
};
|
||||
self.scan(limit, matchers, visit, |a, b| {
|
||||
rank(
|
||||
(a.library, a.score, &a.symbol.name),
|
||||
(b.library, b.score, &b.symbol.name),
|
||||
)
|
||||
.then_with(|| a.symbol.fqn.cmp(&b.symbol.fqn))
|
||||
.then_with(|| a.path.cmp(b.path))
|
||||
.then_with(|| a.symbol.line.cmp(&b.symbol.line))
|
||||
})
|
||||
}
|
||||
|
||||
/// Runs `visit` over every file and returns the best `limit` hits in
|
||||
/// `order`. Large indexes are scanned on [`crate::default_threads`]
|
||||
/// threads, each with its own matcher state from `make` and its own
|
||||
/// best hits, merged at the end; small ones on the calling thread.
|
||||
/// Returns nothing when `make` gives no state (an empty query).
|
||||
fn scan<'a, S, T: Send>(
|
||||
&'a self,
|
||||
limit: usize,
|
||||
make: impl Fn() -> Option<S> + Sync,
|
||||
visit: impl Fn(&mut S, &'a IndexedFile, &mut Vec<T>) + Sync,
|
||||
order: impl Fn(&T, &T) -> Ordering + Sync,
|
||||
) -> Vec<T> {
|
||||
let files: Vec<&IndexedFile> = self.files.values().collect();
|
||||
let work = files.len().saturating_add(self.symbols);
|
||||
let threads = if work < PARALLEL_FROM {
|
||||
1
|
||||
} else {
|
||||
crate::default_threads()
|
||||
};
|
||||
let chunk = files.len().div_ceil(threads).max(1);
|
||||
let run = |part: &[&'a IndexedFile]| {
|
||||
let mut hits = Vec::new();
|
||||
if let Some(mut state) = make() {
|
||||
for file in part {
|
||||
visit(&mut state, file, &mut hits);
|
||||
}
|
||||
}
|
||||
best(&mut hits, limit, &order);
|
||||
hits
|
||||
};
|
||||
let mut hits: Vec<T> = if threads <= 1 {
|
||||
run(&files)
|
||||
} else {
|
||||
std::thread::scope(|scope| {
|
||||
let handles: Vec<_> = files
|
||||
.chunks(chunk)
|
||||
.map(|part| scope.spawn(|| run(part)))
|
||||
.collect();
|
||||
handles
|
||||
.into_iter()
|
||||
.flat_map(|h| h.join().unwrap_or_default())
|
||||
.collect()
|
||||
})
|
||||
};
|
||||
best(&mut hits, limit, &order);
|
||||
hits
|
||||
}
|
||||
}
|
||||
|
||||
/// The ranking order: project before library, higher score first, then
|
||||
/// shorter, then byte order.
|
||||
fn rank(a: (bool, u32, &str), b: (bool, u32, &str)) -> Ordering {
|
||||
a.0.cmp(&b.0)
|
||||
.then_with(|| b.1.cmp(&a.1))
|
||||
.then_with(|| a.2.len().cmp(&b.2.len()))
|
||||
.then_with(|| a.2.cmp(b.2))
|
||||
}
|
||||
|
||||
/// Keeps the `limit` (at most [`MAX_LIMIT`]) first items of `hits` in
|
||||
/// `order`, sorted.
|
||||
fn best<T>(hits: &mut Vec<T>, limit: usize, order: impl Fn(&T, &T) -> Ordering) {
|
||||
let limit = limit.min(MAX_LIMIT);
|
||||
if limit == 0 {
|
||||
hits.clear();
|
||||
return;
|
||||
}
|
||||
if hits.len() > limit {
|
||||
hits.select_nth_unstable_by(limit.saturating_sub(1), &order);
|
||||
hits.truncate(limit);
|
||||
}
|
||||
hits.sort_by(order);
|
||||
}
|
||||
|
||||
/// The matcher configuration for symbol names: the default word
|
||||
/// boundaries (camel case, `\`, `_`, `:` and other punctuation start
|
||||
/// words), and matches near the start score a little higher.
|
||||
fn symbol_config() -> Config {
|
||||
let mut config = Config::DEFAULT;
|
||||
config.prefer_prefix = true;
|
||||
config
|
||||
}
|
||||
|
||||
/// The first [`MAX_QUERY_CHARS`] characters of `query`.
|
||||
fn clamp(query: &str) -> String {
|
||||
query.chars().take(MAX_QUERY_CHARS).collect()
|
||||
}
|
||||
|
||||
/// The file name of a project-relative path.
|
||||
fn file_name(path: &str) -> &str {
|
||||
path.rsplit('/').next().unwrap_or(path)
|
||||
}
|
||||
|
||||
/// Length of `text` in UTF-16 code units.
|
||||
fn utf16_len(text: &str) -> usize {
|
||||
text.chars().map(char::len_utf16).sum()
|
||||
}
|
||||
|
||||
/// Turns sorted character indices into `text` into merged `[start, end)`
|
||||
/// UTF-16 ranges, shifted by `base`.
|
||||
fn ranges(text: &str, indices: &[u32], base: usize) -> Vec<[usize; 2]> {
|
||||
let mut out: Vec<[usize; 2]> = Vec::new();
|
||||
let mut wanted = indices.iter().copied().peekable();
|
||||
let mut at = base;
|
||||
for (i, ch) in text.chars().enumerate() {
|
||||
let width = ch.len_utf16();
|
||||
if wanted
|
||||
.peek()
|
||||
.is_some_and(|&w| usize::try_from(w).is_ok_and(|w| w == i))
|
||||
{
|
||||
wanted.next();
|
||||
match out.last_mut() {
|
||||
Some(range) if range[1] == at => range[1] = at.saturating_add(width),
|
||||
_ => out.push([at, at.saturating_add(width)]),
|
||||
}
|
||||
}
|
||||
at = at.saturating_add(width);
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// A compiled query with the matcher and buffers it needs.
|
||||
struct Fuzzy {
|
||||
/// The query words.
|
||||
pattern: Pattern,
|
||||
/// The matcher and its scratch memory.
|
||||
matcher: Matcher,
|
||||
/// Characters of the current haystack, for non-ASCII text.
|
||||
chars: Vec<char>,
|
||||
/// Match indices, reused.
|
||||
indices: Vec<u32>,
|
||||
}
|
||||
|
||||
impl Fuzzy {
|
||||
/// The query `query` with `config`; `None` for a query without words.
|
||||
fn new(query: &str, config: Config) -> Option<Self> {
|
||||
let pattern = Pattern::new(
|
||||
query,
|
||||
CaseMatching::Smart,
|
||||
Normalization::Smart,
|
||||
AtomKind::Fuzzy,
|
||||
);
|
||||
if pattern.atoms.is_empty() {
|
||||
return None;
|
||||
}
|
||||
Some(Self {
|
||||
pattern,
|
||||
matcher: Matcher::new(config),
|
||||
chars: Vec::new(),
|
||||
indices: Vec::new(),
|
||||
})
|
||||
}
|
||||
|
||||
/// The score of `haystack`, `None` if it does not match.
|
||||
fn score(&mut self, haystack: &str) -> Option<u32> {
|
||||
let Self {
|
||||
pattern,
|
||||
matcher,
|
||||
chars,
|
||||
..
|
||||
} = self;
|
||||
pattern.score(utf32(haystack, chars), matcher)
|
||||
}
|
||||
|
||||
/// The matched character indices in `haystack`, sorted, without
|
||||
/// duplicates; empty if it does not match.
|
||||
fn indices(&mut self, haystack: &str) -> Vec<u32> {
|
||||
let Self {
|
||||
pattern,
|
||||
matcher,
|
||||
chars,
|
||||
indices,
|
||||
} = self;
|
||||
indices.clear();
|
||||
if pattern
|
||||
.indices(utf32(haystack, chars), matcher, indices)
|
||||
.is_none()
|
||||
{
|
||||
return Vec::new();
|
||||
}
|
||||
indices.sort_unstable();
|
||||
indices.dedup();
|
||||
indices.clone()
|
||||
}
|
||||
}
|
||||
|
||||
/// `text` as the matcher's string type, one entry per Unicode scalar
|
||||
/// value, so match indices are character indices (`Utf32Str::new` would
|
||||
/// merge grapheme clusters).
|
||||
fn utf32<'a>(text: &'a str, chars: &'a mut Vec<char>) -> Utf32Str<'a> {
|
||||
if text.is_ascii() {
|
||||
return Utf32Str::Ascii(text.as_bytes());
|
||||
}
|
||||
chars.clear();
|
||||
chars.extend(text.chars());
|
||||
Utf32Str::Unicode(chars)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::SymbolKind;
|
||||
|
||||
fn file(path: &str, symbols: &[(&str, &str, SymbolKind, Option<&str>)]) -> IndexedFile {
|
||||
IndexedFile {
|
||||
path: path.into(),
|
||||
library: crate::is_library(path),
|
||||
symbols: symbols
|
||||
.iter()
|
||||
.map(|(name, fqn, kind, container)| Symbol {
|
||||
name: (*name).into(),
|
||||
fqn: (*fqn).into(),
|
||||
kind: *kind,
|
||||
container: container.map(str::to_string),
|
||||
line: 1,
|
||||
column: 2,
|
||||
})
|
||||
.collect(),
|
||||
}
|
||||
}
|
||||
|
||||
fn sample() -> Index {
|
||||
let mut index = Index::new();
|
||||
index.insert(file(
|
||||
"src/Models/User.php",
|
||||
&[
|
||||
("User", "App\\Models\\User", SymbolKind::Class, None),
|
||||
(
|
||||
"find",
|
||||
"App\\Models\\User::find",
|
||||
SymbolKind::Method,
|
||||
Some("App\\Models\\User"),
|
||||
),
|
||||
(
|
||||
"$name",
|
||||
"App\\Models\\User::$name",
|
||||
SymbolKind::Property,
|
||||
Some("App\\Models\\User"),
|
||||
),
|
||||
],
|
||||
));
|
||||
index.insert(file(
|
||||
"src/user/Repository.php",
|
||||
&[(
|
||||
"Repository",
|
||||
"App\\User\\Repository",
|
||||
SymbolKind::Class,
|
||||
None,
|
||||
)],
|
||||
));
|
||||
index.insert(file(
|
||||
"vendor/acme/lib/src/User.php",
|
||||
&[
|
||||
("User", "Acme\\User", SymbolKind::Class, None),
|
||||
(
|
||||
"find",
|
||||
"Acme\\User::find",
|
||||
SymbolKind::Method,
|
||||
Some("Acme\\User"),
|
||||
),
|
||||
],
|
||||
));
|
||||
index.insert(file("templates/user.html.twig", &[]));
|
||||
index.insert(file(
|
||||
"src/helpers.php",
|
||||
&[("format_user", "format_user", SymbolKind::Function, None)],
|
||||
));
|
||||
index
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn counts_and_removal() {
|
||||
let mut index = sample();
|
||||
assert_eq!(index.len(), 5);
|
||||
assert_eq!(index.symbol_count(), 7);
|
||||
index.insert(file("src/helpers.php", &[]));
|
||||
assert_eq!(index.symbol_count(), 6);
|
||||
assert_eq!(index.remove("src/user"), 1);
|
||||
assert_eq!(index.remove("src/Models/User.php"), 1);
|
||||
assert_eq!(index.remove("src/Models"), 0);
|
||||
assert_eq!(index.symbol_count(), 2);
|
||||
assert!(index.contains("src/helpers.php"));
|
||||
assert_eq!(index.remove(""), 3);
|
||||
assert!(index.is_empty());
|
||||
assert_eq!(index.symbol_count(), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prefix_removal_respects_segments() {
|
||||
let mut index = Index::new();
|
||||
index.insert(file("src/a.php", &[]));
|
||||
index.insert(file("src2/b.php", &[]));
|
||||
index.insert(file("src-x/c.php", &[]));
|
||||
assert_eq!(index.remove("src"), 1);
|
||||
assert_eq!(index.len(), 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn files_rank_names_then_paths_and_project_first() {
|
||||
let index = sample();
|
||||
let paths: Vec<String> = index
|
||||
.find_files("user", 10)
|
||||
.into_iter()
|
||||
.map(|h| h.path)
|
||||
.collect();
|
||||
assert_eq!(
|
||||
paths,
|
||||
vec![
|
||||
"src/Models/User.php",
|
||||
"templates/user.html.twig",
|
||||
"src/user/Repository.php",
|
||||
"vendor/acme/lib/src/User.php",
|
||||
]
|
||||
);
|
||||
let hit = &index.find_files("user", 1)[0];
|
||||
assert_eq!(hit.matches, vec![[11, 15]]);
|
||||
let by_path = index.find_files("models/u", 10);
|
||||
assert_eq!(by_path[0].path, "src/Models/User.php");
|
||||
assert!(index.find_files(" ", 10).is_empty());
|
||||
assert!(index.find_files("user", 0).is_empty());
|
||||
assert!(index.find_files("zzz", 10).is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn classes_and_symbols() {
|
||||
let index = sample();
|
||||
let classes: Vec<(&str, bool)> = index
|
||||
.find_classes("user", 10)
|
||||
.iter()
|
||||
.map(|h| (h.symbol.fqn.as_str(), h.library))
|
||||
.collect();
|
||||
assert_eq!(
|
||||
classes,
|
||||
vec![("App\\Models\\User", false), ("Acme\\User", true)]
|
||||
);
|
||||
let fqn = index.find_classes("\\acme\\us", 10);
|
||||
assert_eq!(fqn.len(), 1);
|
||||
assert_eq!(fqn[0].symbol.fqn, "Acme\\User");
|
||||
let finds: Vec<&str> = index
|
||||
.find_symbols("find", 10)
|
||||
.iter()
|
||||
.map(|h| h.symbol.fqn.as_str())
|
||||
.collect();
|
||||
assert_eq!(finds, vec!["App\\Models\\User::find", "Acme\\User::find"]);
|
||||
let members: Vec<&str> = index
|
||||
.find_symbols("User::", 10)
|
||||
.iter()
|
||||
.map(|h| h.symbol.name.as_str())
|
||||
.collect();
|
||||
assert_eq!(members, vec!["find", "$name", "find"]);
|
||||
let scoped = index.find_symbols("acme\\user::f", 10);
|
||||
assert_eq!(scoped.len(), 1);
|
||||
assert!(
|
||||
index
|
||||
.find_symbols("user", 10)
|
||||
.iter()
|
||||
.all(|h| !h.symbol.kind.is_class_like())
|
||||
);
|
||||
assert!(index.find_symbols("::", 10).is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn match_ranges_count_utf16() {
|
||||
assert_eq!(ranges("aéb", &[0, 1, 2], 0), vec![[0, 3]]);
|
||||
assert_eq!(ranges("a😀b", &[2], 4), vec![[7, 8]]);
|
||||
assert_eq!(ranges("abc", &[0, 2], 0), vec![[0, 1], [2, 3]]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn caps_the_number_of_files() {
|
||||
let mut index = Index::new();
|
||||
for i in 0..MAX_FILES {
|
||||
index.files.insert(format!("f{i}"), IndexedFile::default());
|
||||
}
|
||||
assert!(!index.insert(file("new.php", &[])));
|
||||
assert!(index.insert(file("f1", &[])));
|
||||
}
|
||||
}
|
||||
313
crates/index/src/symbol.rs
Normal file
313
crates/index/src/symbol.rs
Normal file
|
|
@ -0,0 +1,313 @@
|
|||
//! Symbols and indexed files, extracted from tree-sitter outlines.
|
||||
|
||||
use lutra_syntax::{OutlineItem, OutlineKind, Parser};
|
||||
|
||||
use crate::{MAX_PARSE_BYTES, MAX_SYMBOLS_PER_FILE};
|
||||
|
||||
/// What a symbol declares.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
|
||||
pub enum SymbolKind {
|
||||
/// A class.
|
||||
Class,
|
||||
/// An interface.
|
||||
Interface,
|
||||
/// A trait.
|
||||
Trait,
|
||||
/// An enum.
|
||||
Enum,
|
||||
/// A function outside a class.
|
||||
Function,
|
||||
/// A method.
|
||||
Method,
|
||||
/// A property, declared or constructor-promoted.
|
||||
Property,
|
||||
/// A class constant or a `const` outside a class.
|
||||
Constant,
|
||||
/// A `case` of an enum.
|
||||
EnumCase,
|
||||
}
|
||||
|
||||
impl SymbolKind {
|
||||
/// The kind's name as the engine and the UI use it, the same as the
|
||||
/// outline's: `class`, `interface`, `trait`, `enum`, `function`,
|
||||
/// `method`, `property`, `constant` or `enumCase`.
|
||||
pub fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::Class => "class",
|
||||
Self::Interface => "interface",
|
||||
Self::Trait => "trait",
|
||||
Self::Enum => "enum",
|
||||
Self::Function => "function",
|
||||
Self::Method => "method",
|
||||
Self::Property => "property",
|
||||
Self::Constant => "constant",
|
||||
Self::EnumCase => "enumCase",
|
||||
}
|
||||
}
|
||||
|
||||
/// Whether the kind is a class, interface, trait or enum (Go to Class);
|
||||
/// the others are found by Go to Symbol.
|
||||
pub fn is_class_like(self) -> bool {
|
||||
matches!(
|
||||
self,
|
||||
Self::Class | Self::Interface | Self::Trait | Self::Enum
|
||||
)
|
||||
}
|
||||
|
||||
/// The symbol kind of an outline kind; `None` for a namespace.
|
||||
fn from_outline(kind: OutlineKind) -> Option<Self> {
|
||||
Some(match kind {
|
||||
OutlineKind::Namespace => return None,
|
||||
OutlineKind::Class => Self::Class,
|
||||
OutlineKind::Interface => Self::Interface,
|
||||
OutlineKind::Trait => Self::Trait,
|
||||
OutlineKind::Enum => Self::Enum,
|
||||
OutlineKind::Function => Self::Function,
|
||||
OutlineKind::Method => Self::Method,
|
||||
OutlineKind::Property => Self::Property,
|
||||
OutlineKind::Constant => Self::Constant,
|
||||
OutlineKind::EnumCase => Self::EnumCase,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// One declaration.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
|
||||
pub struct Symbol {
|
||||
/// The name as written: `Product`, `find`, `$id` (properties keep the
|
||||
/// `$`), `MAX`.
|
||||
pub name: String,
|
||||
/// The fully qualified name without a leading `\`: `App\Models\Product`
|
||||
/// for a class or function, `App\Models\Product::find` for a member.
|
||||
pub fqn: String,
|
||||
/// What it declares.
|
||||
pub kind: SymbolKind,
|
||||
/// The FQN of the class, interface, trait or enum that declares a
|
||||
/// member; `None` for classes, functions and constants outside a class.
|
||||
pub container: Option<String>,
|
||||
/// 0-based line of the name.
|
||||
pub line: usize,
|
||||
/// Column of the name in UTF-16 code units.
|
||||
pub column: usize,
|
||||
}
|
||||
|
||||
/// A file in the index.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Default)]
|
||||
pub struct IndexedFile {
|
||||
/// Project-relative path with `/` separators.
|
||||
pub path: String,
|
||||
/// Library code: the file is below the root's `vendor/`.
|
||||
pub library: bool,
|
||||
/// Declarations in source order; empty for files that are not parsed.
|
||||
pub symbols: Vec<Symbol>,
|
||||
}
|
||||
|
||||
/// Whether the project-relative `path` is library code (the root's
|
||||
/// `vendor/` or anything below it).
|
||||
pub fn is_library(path: &str) -> bool {
|
||||
path == "vendor" || path.starts_with("vendor/")
|
||||
}
|
||||
|
||||
/// Whether the project-relative `path` is a PHP file to parse.
|
||||
pub(crate) fn is_php(path: &str) -> bool {
|
||||
path.rsplit('/')
|
||||
.next()
|
||||
.and_then(|name| name.rsplit_once('.'))
|
||||
.is_some_and(|(stem, ext)| !stem.is_empty() && ext.eq_ignore_ascii_case("php"))
|
||||
}
|
||||
|
||||
/// Indexes `text` as the file at `path`: parses it when it is a PHP file
|
||||
/// of at most [`MAX_PARSE_BYTES`] and extracts its symbols; other files get
|
||||
/// no symbols.
|
||||
///
|
||||
/// # Parameters
|
||||
/// - `parser`: reused between files.
|
||||
/// - `path`: project-relative path with `/` separators.
|
||||
/// - `text`: the file's text.
|
||||
pub fn index_text(parser: &mut Parser, path: String, text: &str) -> IndexedFile {
|
||||
let library = is_library(&path);
|
||||
let parse = is_php(&path) && text.len() as u64 <= MAX_PARSE_BYTES;
|
||||
let symbols = if parse {
|
||||
let tree = parser.parse(text);
|
||||
symbols_from_outline(&lutra_syntax::outline(&tree))
|
||||
} else {
|
||||
Vec::new()
|
||||
};
|
||||
IndexedFile {
|
||||
path,
|
||||
library,
|
||||
symbols,
|
||||
}
|
||||
}
|
||||
|
||||
/// The symbols declared in an outline, in source order, at most
|
||||
/// [`MAX_SYMBOLS_PER_FILE`].
|
||||
///
|
||||
/// Namespaces give the FQN prefix of what they contain; classes,
|
||||
/// interfaces, traits and enums are the container of their members.
|
||||
/// Anonymous classes and their members are skipped: nothing can navigate to
|
||||
/// them by name.
|
||||
pub fn symbols_from_outline(items: &[OutlineItem]) -> Vec<Symbol> {
|
||||
let mut out = Vec::new();
|
||||
collect(items, "", None, &mut out);
|
||||
out
|
||||
}
|
||||
|
||||
/// Adds the symbols of `items` to `out`.
|
||||
///
|
||||
/// # Parameters
|
||||
/// - `namespace`: the enclosing namespace without leading or trailing `\`;
|
||||
/// empty for the global namespace.
|
||||
/// - `container`: the FQN of the enclosing class-like declaration.
|
||||
fn collect(items: &[OutlineItem], namespace: &str, container: Option<&str>, out: &mut Vec<Symbol>) {
|
||||
for item in items {
|
||||
if out.len() >= MAX_SYMBOLS_PER_FILE {
|
||||
return;
|
||||
}
|
||||
let Some(kind) = SymbolKind::from_outline(item.kind) else {
|
||||
let name = item.name.trim_matches('\\');
|
||||
collect(&item.children, name, None, out);
|
||||
continue;
|
||||
};
|
||||
if item.is_anonymous() || item.name.is_empty() {
|
||||
continue;
|
||||
}
|
||||
let (fqn, owner) = match container {
|
||||
Some(owner) if !kind.is_class_like() && kind != SymbolKind::Function => {
|
||||
(format!("{owner}::{}", item.name), Some(owner.to_string()))
|
||||
}
|
||||
_ => (qualify(namespace, &item.name), None),
|
||||
};
|
||||
let start = item.selection_range.start;
|
||||
out.push(Symbol {
|
||||
name: item.name.clone(),
|
||||
fqn: fqn.clone(),
|
||||
kind,
|
||||
container: owner,
|
||||
line: start.line,
|
||||
column: start.column,
|
||||
});
|
||||
if kind.is_class_like() {
|
||||
collect(&item.children, namespace, Some(&fqn), out);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// `name` qualified with `namespace`: `App\Product`, or `Product` in the
|
||||
/// global namespace.
|
||||
fn qualify(namespace: &str, name: &str) -> String {
|
||||
let name = name.trim_start_matches('\\');
|
||||
if namespace.is_empty() {
|
||||
name.to_string()
|
||||
} else {
|
||||
format!("{namespace}\\{name}")
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn symbols(source: &str) -> Vec<Symbol> {
|
||||
let mut parser = Parser::new().unwrap();
|
||||
index_text(&mut parser, "src/a.php".into(), source).symbols
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn qualifies_classes_members_and_functions() {
|
||||
let found = symbols(
|
||||
"<?php\nnamespace App\\Models;\n\ninterface HasId {}\nfinal class Product implements HasId\n{\n public const MAX = 3;\n private ?string $name = null;\n public function __construct(private int $id) {}\n public function find(int $id): ?self { return null; }\n}\nenum Status: string { case Draft = 'draft'; }\nfunction helper(): void {}\nconst VERSION = 1;\n$x = new class { public function hidden() {} };\n",
|
||||
);
|
||||
let summary: Vec<(String, &str, Option<String>)> = found
|
||||
.iter()
|
||||
.map(|s| (s.fqn.clone(), s.kind.as_str(), s.container.clone()))
|
||||
.collect();
|
||||
let product = Some("App\\Models\\Product".to_string());
|
||||
assert_eq!(
|
||||
summary,
|
||||
vec![
|
||||
("App\\Models\\HasId".into(), "interface", None),
|
||||
("App\\Models\\Product".into(), "class", None),
|
||||
(
|
||||
"App\\Models\\Product::MAX".into(),
|
||||
"constant",
|
||||
product.clone()
|
||||
),
|
||||
(
|
||||
"App\\Models\\Product::$name".into(),
|
||||
"property",
|
||||
product.clone()
|
||||
),
|
||||
(
|
||||
"App\\Models\\Product::__construct".into(),
|
||||
"method",
|
||||
product.clone()
|
||||
),
|
||||
(
|
||||
"App\\Models\\Product::$id".into(),
|
||||
"property",
|
||||
product.clone()
|
||||
),
|
||||
("App\\Models\\Product::find".into(), "method", product),
|
||||
("App\\Models\\Status".into(), "enum", None),
|
||||
(
|
||||
"App\\Models\\Status::Draft".into(),
|
||||
"enumCase",
|
||||
Some("App\\Models\\Status".into())
|
||||
),
|
||||
("App\\Models\\helper".into(), "function", None),
|
||||
("App\\Models\\VERSION".into(), "constant", None),
|
||||
]
|
||||
);
|
||||
let product = found.iter().find(|s| s.name == "Product").unwrap();
|
||||
assert_eq!((product.line, product.column), (4, 12));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn braced_and_global_namespaces() {
|
||||
let found = symbols(
|
||||
"<?php\nnamespace A { class One {} }\nnamespace { class Two {} function three() {} }\n",
|
||||
);
|
||||
let fqns: Vec<&str> = found.iter().map(|s| s.fqn.as_str()).collect();
|
||||
assert_eq!(fqns, vec!["A\\One", "Two", "three"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn only_php_files_are_parsed() {
|
||||
let mut parser = Parser::new().unwrap();
|
||||
let text = "<?php class A {}";
|
||||
assert_eq!(
|
||||
index_text(&mut parser, "a.php".into(), text).symbols.len(),
|
||||
1
|
||||
);
|
||||
assert_eq!(
|
||||
index_text(&mut parser, "A.PHP".into(), text).symbols.len(),
|
||||
1
|
||||
);
|
||||
assert!(
|
||||
index_text(&mut parser, "a.twig".into(), text)
|
||||
.symbols
|
||||
.is_empty()
|
||||
);
|
||||
assert!(
|
||||
index_text(&mut parser, ".php".into(), text)
|
||||
.symbols
|
||||
.is_empty()
|
||||
);
|
||||
let file = index_text(&mut parser, "vendor/x/a.php".into(), text);
|
||||
assert!(file.library);
|
||||
assert!(!is_library("vendors/a.php"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn oversized_files_have_no_symbols() {
|
||||
let mut parser = Parser::new().unwrap();
|
||||
let mut text = String::from("<?php class A {}\n");
|
||||
text.push_str(&"// filler\n".repeat((MAX_PARSE_BYTES as usize) / 10 + 1));
|
||||
assert!(
|
||||
index_text(&mut parser, "a.php".into(), &text)
|
||||
.symbols
|
||||
.is_empty()
|
||||
);
|
||||
}
|
||||
}
|
||||
192
crates/index/src/updater.rs
Normal file
192
crates/index/src/updater.rs
Normal file
|
|
@ -0,0 +1,192 @@
|
|||
//! Incremental updates: re-indexing single files and directories after the
|
||||
//! build, on a background thread.
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::path::PathBuf;
|
||||
use std::sync::mpsc::{Receiver, Sender, channel};
|
||||
|
||||
use lutra_syntax::Parser;
|
||||
use lutra_text::Buffer;
|
||||
|
||||
use crate::build::{guarded_index, index_file};
|
||||
use crate::walk::walk;
|
||||
use crate::{CancelToken, IndexedFile, MAX_FILES, Update};
|
||||
|
||||
/// Most files one directory rescan reports in one [`Update::Files`].
|
||||
const RESCAN_BATCH: usize = 256;
|
||||
|
||||
/// Work for an [`Updater`].
|
||||
#[derive(Debug, Clone)]
|
||||
pub enum Job {
|
||||
/// Re-index one file.
|
||||
File {
|
||||
/// Project-relative path with `/` separators.
|
||||
path: String,
|
||||
/// The text of an open editor buffer to index instead of the file on
|
||||
/// disk; `None` reads the disk, and reports [`Update::Removed`] when
|
||||
/// the file is gone or now excluded by the project's rules.
|
||||
text: Option<Buffer>,
|
||||
},
|
||||
/// Re-index a directory: drop everything below it, then list it again
|
||||
/// from disk.
|
||||
Dir {
|
||||
/// Project-relative path with `/` separators.
|
||||
path: String,
|
||||
},
|
||||
}
|
||||
|
||||
impl Job {
|
||||
/// The path the job is about.
|
||||
fn path(&self) -> &str {
|
||||
match self {
|
||||
Self::File { path, .. } | Self::Dir { path } => path,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// A background thread that runs [`Job`]s one batch at a time and sends
|
||||
/// the results as live [`Update`]s. Jobs queued while it works are
|
||||
/// coalesced: only the last job per path runs, so a burst of keystrokes in
|
||||
/// one buffer parses it once. The thread ends when the updater is dropped
|
||||
/// or `cancel` fires.
|
||||
#[derive(Debug)]
|
||||
pub struct Updater {
|
||||
/// Queues jobs for the thread.
|
||||
jobs: Sender<Job>,
|
||||
}
|
||||
|
||||
impl Updater {
|
||||
/// Starts the thread for the project at `root`.
|
||||
///
|
||||
/// # Parameters
|
||||
/// - `root`: the project root (absolute).
|
||||
/// - `cancel`: stops the thread between jobs.
|
||||
/// - `updates`: where results go, usually the build's channel.
|
||||
///
|
||||
/// # Errors
|
||||
/// The thread could not be started.
|
||||
pub fn spawn(
|
||||
root: PathBuf,
|
||||
cancel: CancelToken,
|
||||
updates: Sender<Update>,
|
||||
) -> std::io::Result<Self> {
|
||||
let (jobs, queue) = channel();
|
||||
std::thread::Builder::new()
|
||||
.name("lutra-index-update".into())
|
||||
.spawn(move || run(&root, &cancel, &queue, &updates))?;
|
||||
Ok(Self { jobs })
|
||||
}
|
||||
|
||||
/// Queues `job`. Returns `false` if the thread has ended.
|
||||
pub fn send(&self, job: Job) -> bool {
|
||||
self.jobs.send(job).is_ok()
|
||||
}
|
||||
}
|
||||
|
||||
/// The updater thread's loop.
|
||||
fn run(
|
||||
root: &std::path::Path,
|
||||
cancel: &CancelToken,
|
||||
queue: &Receiver<Job>,
|
||||
updates: &Sender<Update>,
|
||||
) {
|
||||
let mut parser = Parser::new().ok();
|
||||
while let Ok(first) = queue.recv() {
|
||||
let mut jobs = vec![first];
|
||||
jobs.extend(queue.try_iter());
|
||||
for job in coalesce(jobs) {
|
||||
if cancel.is_cancelled() {
|
||||
return;
|
||||
}
|
||||
let sent = match job {
|
||||
Job::File {
|
||||
path,
|
||||
text: Some(text),
|
||||
} => {
|
||||
let library = crate::is_library(&path);
|
||||
let file = guarded_index(&mut parser, path.clone(), &text.text()).unwrap_or(
|
||||
IndexedFile {
|
||||
path,
|
||||
library,
|
||||
symbols: Vec::new(),
|
||||
},
|
||||
);
|
||||
updates
|
||||
.send(Update::Files {
|
||||
files: vec![file],
|
||||
live: true,
|
||||
})
|
||||
.is_ok()
|
||||
}
|
||||
Job::File { path, text: None } => {
|
||||
rescan(root, &path, false, cancel, &mut parser, updates)
|
||||
}
|
||||
Job::Dir { path } => rescan(root, &path, true, cancel, &mut parser, updates),
|
||||
};
|
||||
if !sent {
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Keeps the last job per path, in the order of those last jobs.
|
||||
fn coalesce(jobs: Vec<Job>) -> Vec<Job> {
|
||||
let mut last: HashMap<String, usize> = HashMap::new();
|
||||
for (i, job) in jobs.iter().enumerate() {
|
||||
last.insert(job.path().to_string(), i);
|
||||
}
|
||||
jobs.into_iter()
|
||||
.enumerate()
|
||||
.filter(|(i, job)| last.get(job.path()) == Some(i))
|
||||
.map(|(_, job)| job)
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Lists `path` from disk with the project's rules and indexes what is
|
||||
/// found. A file that is no longer found is reported removed; a directory
|
||||
/// is always reported removed first, so files gone from it disappear.
|
||||
/// Returns `false` once the receiver is gone.
|
||||
fn rescan(
|
||||
root: &std::path::Path,
|
||||
path: &str,
|
||||
dir: bool,
|
||||
cancel: &CancelToken,
|
||||
parser: &mut Option<Parser>,
|
||||
updates: &Sender<Update>,
|
||||
) -> bool {
|
||||
let walked = walk(root, Some(path), 1, MAX_FILES, cancel);
|
||||
if (dir || walked.files.is_empty())
|
||||
&& updates
|
||||
.send(Update::Removed {
|
||||
path: path.to_string(),
|
||||
})
|
||||
.is_err()
|
||||
{
|
||||
return false;
|
||||
}
|
||||
let mut batch = Vec::new();
|
||||
for found in &walked.files {
|
||||
if cancel.is_cancelled() {
|
||||
return true;
|
||||
}
|
||||
batch.push(index_file(parser, found).0);
|
||||
if batch.len() >= RESCAN_BATCH
|
||||
&& updates
|
||||
.send(Update::Files {
|
||||
files: std::mem::take(&mut batch),
|
||||
live: true,
|
||||
})
|
||||
.is_err()
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
batch.is_empty()
|
||||
|| updates
|
||||
.send(Update::Files {
|
||||
files: batch,
|
||||
live: true,
|
||||
})
|
||||
.is_ok()
|
||||
}
|
||||
193
crates/index/src/walk.rs
Normal file
193
crates/index/src/walk.rs
Normal file
|
|
@ -0,0 +1,193 @@
|
|||
//! The walk over the project: which files the index lists (see the crate
|
||||
//! docs for the rules).
|
||||
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::Mutex;
|
||||
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
|
||||
|
||||
use ignore::{DirEntry, WalkBuilder, WalkState};
|
||||
|
||||
use crate::CancelToken;
|
||||
use crate::symbol::is_php;
|
||||
|
||||
/// A file the walk found.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub(crate) struct Found {
|
||||
/// Project-relative path with `/` separators.
|
||||
pub path: String,
|
||||
/// Absolute path.
|
||||
pub abs: PathBuf,
|
||||
/// Size in bytes for PHP files; 0 for other files, which are not read.
|
||||
pub size: u64,
|
||||
}
|
||||
|
||||
/// What a walk returns.
|
||||
#[derive(Debug, Default)]
|
||||
pub(crate) struct Walked {
|
||||
/// The files, in no particular order.
|
||||
pub files: Vec<Found>,
|
||||
/// More than the limit were found; the rest are left out.
|
||||
pub capped: bool,
|
||||
}
|
||||
|
||||
/// Lists the files below `root`, or only those at or below the
|
||||
/// project-relative `scope` (a file or a directory), with the project's
|
||||
/// rules.
|
||||
///
|
||||
/// # Parameters
|
||||
/// - `root`: the project root (absolute).
|
||||
/// - `scope`: `None` for the whole project.
|
||||
/// - `threads`: walker threads.
|
||||
/// - `limit`: most files to return.
|
||||
/// - `cancel`: stops the walk.
|
||||
///
|
||||
/// Entries that cannot be read are skipped, as the project tree skips
|
||||
/// them.
|
||||
pub(crate) fn walk(
|
||||
root: &Path,
|
||||
scope: Option<&str>,
|
||||
threads: usize,
|
||||
limit: usize,
|
||||
cancel: &CancelToken,
|
||||
) -> Walked {
|
||||
let found = Mutex::new(Vec::new());
|
||||
let count = AtomicUsize::new(0);
|
||||
let capped = AtomicBool::new(false);
|
||||
let scope = scope.map(|s| s.trim_matches('/').to_string());
|
||||
walker(root, scope.clone(), threads)
|
||||
.build_parallel()
|
||||
.run(|| {
|
||||
let found = &found;
|
||||
let count = &count;
|
||||
let capped = &capped;
|
||||
let scope = scope.as_deref();
|
||||
Box::new(move |entry| {
|
||||
if cancel.is_cancelled() || capped.load(Ordering::Relaxed) {
|
||||
return WalkState::Quit;
|
||||
}
|
||||
let Ok(entry) = entry else {
|
||||
return WalkState::Continue;
|
||||
};
|
||||
if !entry.file_type().is_some_and(|t| t.is_file()) {
|
||||
return WalkState::Continue;
|
||||
}
|
||||
let Some(rel) = relative(root, entry.path()) else {
|
||||
return WalkState::Continue;
|
||||
};
|
||||
if scope.is_some_and(|s| !inside(&rel, s)) {
|
||||
return WalkState::Continue;
|
||||
}
|
||||
if count.fetch_add(1, Ordering::Relaxed) >= limit {
|
||||
capped.store(true, Ordering::Relaxed);
|
||||
return WalkState::Quit;
|
||||
}
|
||||
let size = if is_php(&rel) {
|
||||
entry.metadata().map(|m| m.len()).unwrap_or(0)
|
||||
} else {
|
||||
0
|
||||
};
|
||||
if let Ok(mut files) = found.lock() {
|
||||
files.push(Found {
|
||||
path: rel,
|
||||
abs: entry.path().to_path_buf(),
|
||||
size,
|
||||
});
|
||||
}
|
||||
WalkState::Continue
|
||||
})
|
||||
});
|
||||
Walked {
|
||||
files: found.into_inner().unwrap_or_default(),
|
||||
capped: capped.load(Ordering::Relaxed),
|
||||
}
|
||||
}
|
||||
|
||||
/// The walker over `root` with the project's rules, entering only
|
||||
/// directories on the way to or inside `scope`.
|
||||
fn walker(root: &Path, scope: Option<String>, threads: usize) -> WalkBuilder {
|
||||
let mut builder = WalkBuilder::new(root);
|
||||
builder
|
||||
.hidden(false)
|
||||
.parents(false)
|
||||
.ignore(true)
|
||||
.git_ignore(true)
|
||||
.git_exclude(true)
|
||||
.git_global(true)
|
||||
.require_git(false)
|
||||
.follow_links(false)
|
||||
.threads(threads.max(1));
|
||||
let vendor = root.join("vendor");
|
||||
let vendor_is_dir = std::fs::symlink_metadata(&vendor).is_ok_and(|m| m.is_dir());
|
||||
let vendor_in_scope = scope
|
||||
.as_deref()
|
||||
.is_none_or(|s| related("vendor", s) || s.is_empty());
|
||||
if vendor_is_dir && vendor_in_scope {
|
||||
builder.add(&vendor);
|
||||
}
|
||||
let root = root.to_path_buf();
|
||||
builder.filter_entry(move |entry| keep_entry(&root, &vendor, scope.as_deref(), entry));
|
||||
builder
|
||||
}
|
||||
|
||||
/// Whether the walk enters (directory) or visits (file) `entry`: `.git`
|
||||
/// and `node_modules` are skipped, the root's `vendor` is reached through
|
||||
/// its own walk root, and with a scope only directories on the way to or
|
||||
/// inside it are entered.
|
||||
fn keep_entry(root: &Path, vendor: &Path, scope: Option<&str>, entry: &DirEntry) -> bool {
|
||||
let name = entry.file_name();
|
||||
if name == ".git" {
|
||||
return false;
|
||||
}
|
||||
let is_dir = entry.file_type().is_some_and(|t| t.is_dir());
|
||||
if is_dir && name == "node_modules" {
|
||||
return false;
|
||||
}
|
||||
if entry.depth() == 1 && entry.path() == vendor {
|
||||
return false;
|
||||
}
|
||||
if is_dir
|
||||
&& let Some(scope) = scope
|
||||
&& let Some(rel) = relative(root, entry.path())
|
||||
{
|
||||
return related(&rel, scope);
|
||||
}
|
||||
true
|
||||
}
|
||||
|
||||
/// Whether `path` is `scope` or below it.
|
||||
fn inside(path: &str, scope: &str) -> bool {
|
||||
scope.is_empty()
|
||||
|| path == scope
|
||||
|| path
|
||||
.strip_prefix(scope)
|
||||
.is_some_and(|rest| rest.starts_with('/'))
|
||||
}
|
||||
|
||||
/// Whether the directory `dir` is on the way to `scope` or inside it.
|
||||
fn related(dir: &str, scope: &str) -> bool {
|
||||
inside(dir, scope) || inside(scope, dir)
|
||||
}
|
||||
|
||||
/// `path` relative to `root` with `/` separators; `None` outside the root
|
||||
/// or for a name that is not valid Unicode.
|
||||
fn relative(root: &Path, path: &Path) -> Option<String> {
|
||||
let rel = path.strip_prefix(root).ok()?;
|
||||
let parts: Option<Vec<&str>> = rel.components().map(|c| c.as_os_str().to_str()).collect();
|
||||
Some(parts?.join("/"))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn scope_relations() {
|
||||
assert!(inside("src/a.php", "src"));
|
||||
assert!(inside("src", "src"));
|
||||
assert!(!inside("src2/a.php", "src"));
|
||||
assert!(inside("anything", ""));
|
||||
assert!(related("src", "src/deep/a.php"));
|
||||
assert!(related("src/deep/x", "src/deep"));
|
||||
assert!(!related("lib", "src/a.php"));
|
||||
}
|
||||
}
|
||||
222
crates/index/tests/index.rs
Normal file
222
crates/index/tests/index.rs
Normal file
|
|
@ -0,0 +1,222 @@
|
|||
//! Building the index of a project on disk, incremental updates, and
|
||||
//! cancellation.
|
||||
|
||||
#![allow(
|
||||
clippy::unwrap_used,
|
||||
clippy::expect_used,
|
||||
clippy::indexing_slicing,
|
||||
clippy::panic
|
||||
)]
|
||||
|
||||
use std::fs;
|
||||
use std::path::Path;
|
||||
use std::sync::Mutex;
|
||||
use std::sync::mpsc::{Receiver, channel};
|
||||
use std::time::Duration;
|
||||
|
||||
use lutra_index::{CancelToken, Index, Job, Update, Updater, build, spawn_build};
|
||||
|
||||
/// Builds the index of `root` and applies every update.
|
||||
fn build_index(root: &Path, threads: usize) -> (Index, lutra_index::Summary, Vec<(usize, usize)>) {
|
||||
let updates = Mutex::new(Vec::new());
|
||||
let summary = build(root, threads, &CancelToken::new(), &|u| {
|
||||
updates.lock().unwrap().push(u)
|
||||
});
|
||||
let mut index = Index::new();
|
||||
let mut progress = Vec::new();
|
||||
for update in updates.into_inner().unwrap() {
|
||||
match update {
|
||||
Update::Files { files, live } => {
|
||||
assert!(!live);
|
||||
for file in files {
|
||||
assert!(index.insert(file));
|
||||
}
|
||||
}
|
||||
Update::Progress { done, total } => progress.push((done, total)),
|
||||
other => panic!("unexpected {other:?}"),
|
||||
}
|
||||
}
|
||||
(index, summary, progress)
|
||||
}
|
||||
|
||||
/// Applies updates from `rx` until one batch arrives and `rx` is quiet.
|
||||
fn apply_until_quiet(index: &mut Index, rx: &Receiver<Update>) {
|
||||
let first = rx.recv_timeout(Duration::from_secs(10)).expect("an update");
|
||||
let mut pending = vec![first];
|
||||
while let Ok(next) = rx.recv_timeout(Duration::from_millis(300)) {
|
||||
pending.push(next);
|
||||
}
|
||||
for update in pending {
|
||||
match update {
|
||||
Update::Files { files, live } => {
|
||||
assert!(live);
|
||||
for file in files {
|
||||
index.insert(file);
|
||||
}
|
||||
}
|
||||
Update::Removed { path } => {
|
||||
index.remove(&path);
|
||||
}
|
||||
other => panic!("unexpected {other:?}"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn write(root: &Path, rel: &str, text: &str) {
|
||||
let path = root.join(rel);
|
||||
fs::create_dir_all(path.parent().unwrap()).unwrap();
|
||||
fs::write(path, text).unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn builds_with_the_project_rules() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let project = lutra_testkit::generate_php_project(dir.path(), 400);
|
||||
write(
|
||||
dir.path(),
|
||||
"src/Broken.php",
|
||||
"<?php\nclass Broken {\n public function ok() {}\n public function bad() { $x = ; }\n",
|
||||
);
|
||||
write(dir.path(), "src/Latin1.php", "<?php\nclass Caf\u{e9} {}\n");
|
||||
fs::write(
|
||||
dir.path().join("src/Bad.php"),
|
||||
b"<?php\nclass Bad\xff {}\nclass Fine {}\n",
|
||||
)
|
||||
.unwrap();
|
||||
write(dir.path(), ".git/Hidden.php", "<?php class InGit {}");
|
||||
for threads in [1, 4] {
|
||||
let (index, summary, progress) = build_index(dir.path(), threads);
|
||||
assert_eq!(index.len(), project.indexed_files + 3);
|
||||
assert_eq!(summary.files, index.len());
|
||||
assert_eq!(summary.symbols, index.symbol_count());
|
||||
assert_eq!(summary.parsed, project.php_files + 3);
|
||||
assert!(!summary.capped && !summary.cancelled);
|
||||
let (done, total) = *progress.last().unwrap();
|
||||
assert_eq!((done, total), (index.len(), index.len()));
|
||||
assert!(progress.windows(2).all(|w| w[0].0 <= w[1].0));
|
||||
assert!(index.get("var/cache/Cached0.php").is_none(), "gitignored");
|
||||
assert!(
|
||||
index.get("node_modules/pkg/Hidden0.php").is_none(),
|
||||
"excluded"
|
||||
);
|
||||
assert!(index.get(".git/Hidden.php").is_none());
|
||||
let vendor = index
|
||||
.get("vendor/acme/framework/src/Component000/Widget0.php")
|
||||
.unwrap();
|
||||
assert!(vendor.library, "vendor is walked although gitignored");
|
||||
let classes = index.find_classes("Service1", 200);
|
||||
let service = classes
|
||||
.iter()
|
||||
.find(|h| h.symbol.fqn == "App\\Module000\\Service1")
|
||||
.unwrap();
|
||||
assert_eq!(service.path, "src/Module000/Service1.php");
|
||||
assert_eq!((service.symbol.line, service.symbol.column), (12, 6));
|
||||
assert_eq!(index.find_classes("Broken", 5)[0].symbol.name, "Broken");
|
||||
assert!(!index.find_symbols("Broken::ok", 5).is_empty());
|
||||
assert_eq!(index.find_classes("Café", 5)[0].symbol.name, "Café");
|
||||
assert!(!index.find_classes("Fine", 5).is_empty());
|
||||
let classes_found = index.find_classes("", 10);
|
||||
assert!(classes_found.is_empty());
|
||||
let all: usize = index
|
||||
.find_classes("e", 1_000)
|
||||
.iter()
|
||||
.filter(|h| h.symbol.kind.is_class_like())
|
||||
.count();
|
||||
assert!(all > 0);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn spawned_build_ends_with_ready_and_cancels() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let project = lutra_testkit::generate_php_project(dir.path(), 300);
|
||||
let (tx, rx) = channel();
|
||||
spawn_build(dir.path().to_path_buf(), 2, CancelToken::new(), tx).unwrap();
|
||||
let mut files = 0;
|
||||
let summary = loop {
|
||||
match rx.recv_timeout(Duration::from_secs(30)).unwrap() {
|
||||
Update::Files { files: batch, .. } => files += batch.len(),
|
||||
Update::Ready(summary) => break summary,
|
||||
_ => {}
|
||||
}
|
||||
};
|
||||
assert_eq!(files, project.indexed_files);
|
||||
assert_eq!(summary.files, files);
|
||||
assert!(rx.recv_timeout(Duration::from_millis(200)).is_err());
|
||||
|
||||
let cancel = CancelToken::new();
|
||||
cancel.cancel();
|
||||
let (tx, rx) = channel();
|
||||
spawn_build(dir.path().to_path_buf(), 2, cancel, tx).unwrap();
|
||||
let summary = loop {
|
||||
if let Update::Ready(summary) = rx.recv_timeout(Duration::from_secs(30)).unwrap() {
|
||||
break summary;
|
||||
}
|
||||
};
|
||||
assert!(summary.cancelled);
|
||||
assert!(summary.files < project.indexed_files);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn updater_follows_disk_and_buffers() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let root = dir.path();
|
||||
write(root, ".gitignore", "*.log\n/ignored/\n");
|
||||
write(root, "src/A.php", "<?php class Alpha {}");
|
||||
let (mut index, _, _) = build_index(root, 2);
|
||||
assert_eq!(index.find_classes("Alpha", 5).len(), 1);
|
||||
let (tx, rx) = channel();
|
||||
let updater = Updater::spawn(root.to_path_buf(), CancelToken::new(), tx).unwrap();
|
||||
|
||||
write(root, "src/A.php", "<?php class Alpha {} class Beta {}");
|
||||
updater.send(Job::File {
|
||||
path: "src/A.php".into(),
|
||||
text: None,
|
||||
});
|
||||
apply_until_quiet(&mut index, &rx);
|
||||
assert_eq!(index.get("src/A.php").unwrap().symbols.len(), 2);
|
||||
|
||||
let buffer = lutra_text::Buffer::new("<?php class Gamma { public function go() {} }");
|
||||
for _ in 0..5 {
|
||||
updater.send(Job::File {
|
||||
path: "src/A.php".into(),
|
||||
text: Some(buffer.clone()),
|
||||
});
|
||||
}
|
||||
apply_until_quiet(&mut index, &rx);
|
||||
assert!(index.find_classes("Beta", 5).is_empty());
|
||||
assert_eq!(index.find_symbols("Gamma::go", 5).len(), 1);
|
||||
|
||||
write(root, "src/sub/deep/B.php", "<?php class Delta {}");
|
||||
write(root, "src/sub/notes.log", "ignored");
|
||||
updater.send(Job::Dir {
|
||||
path: "src/sub".into(),
|
||||
});
|
||||
apply_until_quiet(&mut index, &rx);
|
||||
assert!(index.contains("src/sub/deep/B.php"));
|
||||
assert!(!index.contains("src/sub/notes.log"));
|
||||
|
||||
write(root, "ignored/C.php", "<?php class Epsilon {}");
|
||||
updater.send(Job::File {
|
||||
path: "ignored/C.php".into(),
|
||||
text: None,
|
||||
});
|
||||
apply_until_quiet(&mut index, &rx);
|
||||
assert!(!index.contains("ignored/C.php"));
|
||||
|
||||
fs::remove_dir_all(root.join("src/sub")).unwrap();
|
||||
updater.send(Job::Dir {
|
||||
path: "src/sub".into(),
|
||||
});
|
||||
apply_until_quiet(&mut index, &rx);
|
||||
assert!(!index.contains("src/sub/deep/B.php"));
|
||||
|
||||
fs::remove_file(root.join("src/A.php")).unwrap();
|
||||
updater.send(Job::File {
|
||||
path: "src/A.php".into(),
|
||||
text: None,
|
||||
});
|
||||
apply_until_quiet(&mut index, &rx);
|
||||
assert!(!index.contains("src/A.php"));
|
||||
assert_eq!(index.symbol_count(), 0);
|
||||
}
|
||||
78
crates/index/tests/perf.rs
Normal file
78
crates/index/tests/perf.rs
Normal file
|
|
@ -0,0 +1,78 @@
|
|||
//! Index timing on a generated 20,000-file project. Ignored by default; run
|
||||
//! `cargo test --release -p lutra-index --test perf -- --ignored --nocapture`.
|
||||
|
||||
#![allow(
|
||||
clippy::unwrap_used,
|
||||
clippy::expect_used,
|
||||
clippy::indexing_slicing,
|
||||
clippy::panic
|
||||
)]
|
||||
|
||||
use std::time::Instant;
|
||||
|
||||
use lutra_index::{CancelToken, Index, Update, build, default_threads};
|
||||
|
||||
#[test]
|
||||
#[ignore = "measurement; run in release"]
|
||||
fn index_20k_files() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let project = lutra_testkit::generate_php_project(dir.path(), 20_000);
|
||||
println!(
|
||||
"generated: {} indexed files, {} PHP ({} vendor), {:.1} MB PHP, {} classes",
|
||||
project.indexed_files,
|
||||
project.php_files,
|
||||
project.vendor_files,
|
||||
project.php_bytes as f64 / 1e6,
|
||||
project.classes
|
||||
);
|
||||
let threads = default_threads();
|
||||
for round in 0..3 {
|
||||
let index = std::sync::Mutex::new(Index::new());
|
||||
let mut first_batch = None;
|
||||
let started = Instant::now();
|
||||
let first = std::sync::Mutex::new(&mut first_batch);
|
||||
let summary = build(dir.path(), threads, &CancelToken::new(), &|u| {
|
||||
if let Update::Files { files, .. } = u {
|
||||
let mut first = first.lock().unwrap();
|
||||
if first.is_none() {
|
||||
**first = Some(started.elapsed());
|
||||
}
|
||||
let mut index = index.lock().unwrap();
|
||||
for f in files {
|
||||
index.insert(f);
|
||||
}
|
||||
}
|
||||
});
|
||||
let index = index.into_inner().unwrap();
|
||||
assert_eq!(index.len(), project.indexed_files);
|
||||
let time = |f: &dyn Fn() -> usize| {
|
||||
let mut runs: Vec<f64> = (0..7)
|
||||
.map(|_| {
|
||||
let t = Instant::now();
|
||||
assert!(f() > 0);
|
||||
t.elapsed().as_secs_f64() * 1e3
|
||||
})
|
||||
.collect();
|
||||
runs.sort_by(f64::total_cmp);
|
||||
(runs[0], runs[3])
|
||||
};
|
||||
let class = time(&|| index.find_classes("Serv12", 50).len());
|
||||
let symbol = time(&|| index.find_symbols("handle1", 50).len());
|
||||
let member = time(&|| index.find_symbols("Service12::hand", 50).len());
|
||||
let file = time(&|| index.find_files("wid12", 50).len());
|
||||
println!(
|
||||
"round {round}: {threads} workers, walk {:.0} ms, first batch {:.0} ms, total {:.0} ms, {} files, {} symbols, {} parsed ({:.1} MB)",
|
||||
summary.walk.as_secs_f64() * 1e3,
|
||||
first_batch.unwrap().as_secs_f64() * 1e3,
|
||||
summary.elapsed.as_secs_f64() * 1e3,
|
||||
summary.files,
|
||||
summary.symbols,
|
||||
summary.parsed,
|
||||
summary.parsed_bytes as f64 / 1e6,
|
||||
);
|
||||
println!(
|
||||
" queries (min / median of 7): goto.class {:.1} / {:.1} ms, goto.symbol {:.1} / {:.1} ms, Class::member {:.1} / {:.1} ms, goto.file {:.1} / {:.1} ms",
|
||||
class.0, class.1, symbol.0, symbol.1, member.0, member.1, file.0, file.1
|
||||
);
|
||||
}
|
||||
}
|
||||
88
crates/index/tests/properties.rs
Normal file
88
crates/index/tests/properties.rs
Normal file
|
|
@ -0,0 +1,88 @@
|
|||
//! Property tests: random queries and random file contents never panic, and
|
||||
//! query results respect the limit and the ranking order.
|
||||
|
||||
#![allow(
|
||||
clippy::unwrap_used,
|
||||
clippy::expect_used,
|
||||
clippy::indexing_slicing,
|
||||
clippy::panic
|
||||
)]
|
||||
|
||||
use lutra_index::{Index, IndexedFile, index_text};
|
||||
use lutra_syntax::Parser;
|
||||
use proptest::prelude::*;
|
||||
|
||||
/// A small index with project and library files, unicode names included.
|
||||
fn sample() -> Index {
|
||||
let mut parser = Parser::new().unwrap();
|
||||
let mut index = Index::new();
|
||||
let files = [
|
||||
(
|
||||
"src/Models/User.php",
|
||||
"<?php namespace App\\Models; class User { public function find() {} private $név; const MAX = 1; }",
|
||||
),
|
||||
(
|
||||
"src/Ünïcode/Grüße.php",
|
||||
"<?php namespace Ü; final class Grüße { public function sägen() {} }",
|
||||
),
|
||||
(
|
||||
"src/helpers.php",
|
||||
"<?php function format_user() {} const LIMIT = 3;",
|
||||
),
|
||||
(
|
||||
"vendor/acme/User.php",
|
||||
"<?php namespace Acme; interface User { public function find(); } enum E: int { case A = 1; }",
|
||||
),
|
||||
("templates/a😀b.twig", ""),
|
||||
];
|
||||
for (path, text) in files {
|
||||
index.insert(index_text(&mut parser, path.into(), text));
|
||||
}
|
||||
index
|
||||
}
|
||||
|
||||
proptest! {
|
||||
#![proptest_config(ProptestConfig::with_cases(512))]
|
||||
|
||||
#[test]
|
||||
fn random_queries_never_panic(query in "\\PC{0,40}|[\\\\/:$ _a-zA-Z]{0,12}", limit in 0usize..2_000) {
|
||||
let index = sample();
|
||||
let files = index.find_files(&query, limit);
|
||||
let classes = index.find_classes(&query, limit);
|
||||
let symbols = index.find_symbols(&query, limit);
|
||||
let cap = limit.min(lutra_index::MAX_LIMIT);
|
||||
prop_assert!(files.len() <= cap && classes.len() <= cap && symbols.len() <= cap);
|
||||
for pair in files.windows(2) {
|
||||
prop_assert!(pair[0].library <= pair[1].library);
|
||||
if pair[0].library == pair[1].library {
|
||||
prop_assert!(pair[0].score >= pair[1].score);
|
||||
}
|
||||
}
|
||||
for hit in &files {
|
||||
let len: usize = hit.path.encode_utf16().count();
|
||||
for range in &hit.matches {
|
||||
prop_assert!(range[0] < range[1] && range[1] <= len);
|
||||
}
|
||||
prop_assert!(hit.matches.windows(2).all(|w| w[0][1] < w[1][0]));
|
||||
}
|
||||
prop_assert!(classes.iter().all(|h| h.symbol.kind.is_class_like()));
|
||||
prop_assert!(symbols.iter().all(|h| !h.symbol.kind.is_class_like()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn random_sources_never_panic(bytes in proptest::collection::vec(any::<u8>(), 0..2_000)) {
|
||||
let mut parser = Parser::new().unwrap();
|
||||
let mut text = String::from("<?php ");
|
||||
text.push_str(&String::from_utf8_lossy(&bytes));
|
||||
let file = index_text(&mut parser, "src/x.php".into(), &text);
|
||||
let mut index = Index::new();
|
||||
index.insert(file.clone());
|
||||
for symbol in &file.symbols {
|
||||
prop_assert!(!symbol.name.is_empty());
|
||||
let _ = index.find_symbols(&symbol.name, 10);
|
||||
let _ = index.find_classes(&symbol.name, 10);
|
||||
}
|
||||
index.insert(IndexedFile { path: "src/x.php".into(), library: false, symbols: Vec::new() });
|
||||
prop_assert_eq!(index.symbol_count(), 0);
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue