M2.5: symbol index and go-to commands

crates/index: walks the project with the search/VFS rules (vendor as
library), parses .php files from tree-sitter outlines on a worker pool
off the engine lock, updates incrementally from disk and buffers through
an updater thread, caps at 200,000 files, and answers fuzzy file, class
and symbol queries (nucleo-matcher, MPL-2.0 as a separate crate).

Engine: index.status, goto.file, goto.class, goto.symbol; events
indexProgress and indexReady drained by the pump; the index starts on
project.open and is cancelled on close. Tests: unit, incremental,
property (random queries and sources), engine and TCP control tests,
release timing on a generated 20k-file project (3.9-4.2 s).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
danny 2026-10-17 12:53:33 +02:00
parent f13a49064d
commit 053441d066
26 changed files with 3455 additions and 9 deletions

257
crates/index/src/build.rs Normal file
View file

@ -0,0 +1,257 @@
//! Building the index of a whole project in parallel.
use std::io::Read;
use std::panic::{AssertUnwindSafe, catch_unwind};
use std::path::{Path, PathBuf};
use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering};
use std::sync::mpsc::Sender;
use std::time::{Duration, Instant};
use lutra_syntax::Parser;
use crate::symbol::{is_library, is_php};
use crate::walk::{Found, walk};
use crate::{
BATCH_FILES, CancelToken, IndexedFile, MAX_FILES, MAX_PARSE_BYTES, Summary, Update, index_text,
};
/// Longest a worker keeps indexed files before sending them.
const FLUSH_AFTER: Duration = Duration::from_millis(100);
/// Files a worker collects before sending them.
const WORKER_BATCH: usize = 64;
/// Walks the project at `root` and indexes it, reporting [`Update::Files`]
/// batches and [`Update::Progress`] through `send` while it runs (from the
/// calling thread and the workers). Blocks until every file is indexed or
/// `cancel` fires. Does not send [`Update::Ready`]; the caller turns the
/// returned summary into it.
///
/// Files without symbols (not PHP, or larger than [`MAX_PARSE_BYTES`]) are
/// reported right after the walk. PHP files are then parsed on `threads`
/// workers, project files before library files and larger files first
/// within each group, so the longest parses start early. A file that
/// cannot be read is listed without symbols.
pub fn build(
root: &Path,
threads: usize,
cancel: &CancelToken,
send: &(dyn Fn(Update) + Sync),
) -> Summary {
let started = Instant::now();
let walked = walk(root, None, threads, MAX_FILES, cancel);
let walk_time = started.elapsed();
let total = walked.files.len();
let (mut php, rest): (Vec<Found>, Vec<Found>) = walked
.files
.into_iter()
.partition(|f| is_php(&f.path) && f.size <= MAX_PARSE_BYTES);
let mut done = 0usize;
let mut batch = Vec::with_capacity(BATCH_FILES);
for found in rest {
batch.push(IndexedFile {
library: is_library(&found.path),
path: found.path,
symbols: Vec::new(),
});
if batch.len() >= BATCH_FILES {
done = done.saturating_add(batch.len());
send(Update::Files {
files: std::mem::take(&mut batch),
live: false,
});
}
}
done = done.saturating_add(batch.len());
if !batch.is_empty() {
send(Update::Files {
files: batch,
live: false,
});
}
send(Update::Progress { done, total });
php.sort_by(|a, b| {
is_library(&a.path)
.cmp(&is_library(&b.path))
.then(b.size.cmp(&a.size))
});
let shared = Shared {
files: &php,
next: AtomicUsize::new(0),
done: AtomicUsize::new(done),
total,
symbols: AtomicUsize::new(0),
parsed: AtomicUsize::new(0),
parsed_bytes: AtomicU64::new(0),
cancel,
send,
};
let workers = threads.clamp(1, php.len().max(1));
std::thread::scope(|scope| {
for _ in 0..workers {
scope.spawn(|| shared.work());
}
});
Summary {
files: shared.done.load(Ordering::Relaxed),
symbols: shared.symbols.load(Ordering::Relaxed),
parsed: shared.parsed.load(Ordering::Relaxed),
parsed_bytes: shared.parsed_bytes.load(Ordering::Relaxed),
capped: walked.capped,
cancelled: cancel.is_cancelled(),
walk: walk_time,
elapsed: started.elapsed(),
}
}
/// Runs [`build`] on a thread of its own and sends its updates, then
/// [`Update::Ready`], over `updates`. Updates sent after the receiver is
/// gone are dropped; the caller cancels the build when it stops listening.
///
/// # Errors
/// The thread could not be started.
pub fn spawn_build(
root: PathBuf,
threads: usize,
cancel: CancelToken,
updates: Sender<Update>,
) -> std::io::Result<()> {
std::thread::Builder::new()
.name("lutra-index".into())
.spawn(move || {
let send = |update| {
let _ = updates.send(update);
};
let summary = build(&root, threads, &cancel, &send);
send(Update::Ready(summary));
})
.map(|_| ())
}
/// State the workers of one build share.
struct Shared<'a> {
/// PHP files to parse, in order.
files: &'a [Found],
/// Index of the next file to take.
next: AtomicUsize,
/// Files indexed and sent.
done: AtomicUsize,
/// Files found by the walk.
total: usize,
/// Symbols found.
symbols: AtomicUsize,
/// Files parsed.
parsed: AtomicUsize,
/// Bytes parsed.
parsed_bytes: AtomicU64,
/// Stops the workers.
cancel: &'a CancelToken,
/// Receives the batches.
send: &'a (dyn Fn(Update) + Sync),
}
impl Shared<'_> {
/// One worker: takes files until none are left or the build is
/// cancelled, and sends them in batches.
fn work(&self) {
let mut parser = Parser::new().ok();
let mut batch = Vec::with_capacity(WORKER_BATCH);
let mut last = Instant::now();
while !self.cancel.is_cancelled() {
let i = self.next.fetch_add(1, Ordering::Relaxed);
let Some(found) = self.files.get(i) else {
break;
};
let (file, bytes) = index_file(&mut parser, found);
if let Some(bytes) = bytes {
self.parsed.fetch_add(1, Ordering::Relaxed);
self.parsed_bytes.fetch_add(bytes, Ordering::Relaxed);
}
self.symbols
.fetch_add(file.symbols.len(), Ordering::Relaxed);
batch.push(file);
if batch.len() >= WORKER_BATCH || last.elapsed() >= FLUSH_AFTER {
self.flush(&mut batch);
last = Instant::now();
}
}
self.flush(&mut batch);
}
/// Sends `batch` and the progress after it.
fn flush(&self, batch: &mut Vec<IndexedFile>) {
if batch.is_empty() {
return;
}
let count = batch.len();
(self.send)(Update::Files {
files: std::mem::take(batch),
live: false,
});
let done = self
.done
.fetch_add(count, Ordering::Relaxed)
.saturating_add(count);
(self.send)(Update::Progress {
done,
total: self.total,
});
}
}
/// Reads and indexes one file. Returns the file and the bytes parsed,
/// `None` when it was listed without symbols: it is not a PHP file, could
/// not be read, grew past [`MAX_PARSE_BYTES`], no parser is available, or
/// parsing panicked.
pub(crate) fn index_file(parser: &mut Option<Parser>, found: &Found) -> (IndexedFile, Option<u64>) {
let empty = || IndexedFile {
path: found.path.clone(),
library: is_library(&found.path),
symbols: Vec::new(),
};
if !is_php(&found.path) {
return (empty(), None);
}
let Some(text) = read_source(&found.abs) else {
return (empty(), None);
};
match guarded_index(parser, found.path.clone(), &text) {
Some(file) => (file, Some(text.len() as u64)),
None => (empty(), None),
}
}
/// [`index_text`] with a guard: `None` when no parser is available or
/// parsing panicked (a bug in the parser; the parser is replaced so the
/// next file gets a fresh one, and the index goes on without this file's
/// symbols).
pub(crate) fn guarded_index(
parser: &mut Option<Parser>,
path: String,
text: &str,
) -> Option<IndexedFile> {
let p = parser.as_mut()?;
match catch_unwind(AssertUnwindSafe(|| index_text(p, path, text))) {
Ok(file) => Some(file),
Err(_) => {
*parser = Parser::new().ok();
None
}
}
}
/// The text of the source file at `path`: at most [`MAX_PARSE_BYTES`],
/// without a UTF-8 byte order mark, invalid UTF-8 replaced. `None` if it
/// cannot be read or is larger.
pub(crate) fn read_source(path: &Path) -> Option<String> {
let file = std::fs::File::open(path).ok()?;
let mut bytes = Vec::new();
file.take(MAX_PARSE_BYTES.saturating_add(1))
.read_to_end(&mut bytes)
.ok()?;
if bytes.len() as u64 > MAX_PARSE_BYTES {
return None;
}
let body = bytes.strip_prefix(b"\xEF\xBB\xBF").unwrap_or(&bytes);
Some(String::from_utf8_lossy(body).into_owned())
}

View file

@ -1,4 +1,145 @@
//! Symbol index for Lutra (layer L2).
//!
//! Stores declarations and usages for go-to and find-usages, starting with
//! symbols extracted from tree-sitter trees.
//! Lists every file of a project and the declarations of its PHP files
//! (classes, interfaces, traits, enums, functions, methods, properties,
//! constants, enum cases), extracted from tree-sitter outlines
//! ([`lutra_syntax::outline`]), and answers fuzzy go-to queries over them.
//!
//! # Which files
//! The walk follows the rules of `lutra-vfs` and `lutra-search`:
//! `.gitignore`, `.ignore`, `.git/info/exclude` and the global git excludes
//! apply without a git repository; `.git` and every `node_modules` directory
//! are skipped; the root's `vendor/` is walked even when an ignore file
//! lists it and its files are marked as library code; dot files are listed;
//! symbolic links are not followed. Every file is listed for
//! [`Index::find_files`]; only `.php` files are parsed, and only those up to
//! [`MAX_PARSE_BYTES`]. At most [`MAX_FILES`] files are indexed.
//!
//! # Building
//! [`build`] walks the project, then parses the PHP files on a pool of
//! worker threads (one parser each, one file in memory each), project files
//! before library files and larger files first within each group. It
//! reports [`Update`]s through a callback: batches of indexed files,
//! progress, and a final [`Summary`]. [`spawn_build`] runs it on a thread of
//! its own and sends the updates over a channel. [`Updater`] re-indexes
//! single files and directories later, from disk or from an editor buffer.
//! A [`CancelToken`] stops both between files.
//!
//! # Robustness
//! Source files are untrusted: reads are capped, invalid UTF-8 is replaced,
//! broken PHP yields the declarations that could be recognised, symbols per
//! file are capped ([`MAX_SYMBOLS_PER_FILE`]), and a panic while parsing one
//! file (a bug) loses that file's symbols, not the index.
mod build;
mod query;
mod symbol;
mod updater;
mod walk;
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, Ordering};
use std::time::Duration;
pub use build::{build, spawn_build};
pub use query::{FileHit, Index, MAX_LIMIT, MAX_QUERY_CHARS, SymbolHit};
pub use symbol::{IndexedFile, Symbol, SymbolKind, index_text, is_library, symbols_from_outline};
pub use updater::{Job, Updater};
/// Most files one index holds.
pub const MAX_FILES: usize = 200_000;
/// Largest PHP file parsed, in bytes (2 MiB). Larger files are listed for
/// [`Index::find_files`] without symbols: tree-sitter-php parses about
/// 2–3 MB/s, so one such file would hold a worker for a second or more, and
/// files this size are generated code rather than declarations to navigate.
pub const MAX_PARSE_BYTES: u64 = 2 * 1024 * 1024;
/// Most symbols kept from one file; the rest are dropped.
pub const MAX_SYMBOLS_PER_FILE: usize = 20_000;
/// Most worker threads of one build.
pub const MAX_THREADS: usize = 32;
/// Files per [`Update::Files`] batch of a build.
pub const BATCH_FILES: usize = 256;
/// What a build or an [`Updater`] reports.
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum Update {
/// Indexed files that replace any earlier entry with the same path.
Files {
/// The files.
files: Vec<IndexedFile>,
/// From an [`Updater`] (a change after the build started), so newer
/// than anything the build reports for the same path.
live: bool,
},
/// A file or directory is gone: drop `path` and everything below it.
Removed {
/// Project-relative path with `/` separators.
path: String,
},
/// Build progress: `done` of `total` files are indexed.
Progress {
/// Files indexed so far.
done: usize,
/// Files found by the walk.
total: usize,
},
/// The build ended; nothing from it follows.
Ready(Summary),
}
/// How a build ended.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
pub struct Summary {
/// Files indexed.
pub files: usize,
/// Symbols found in them.
pub symbols: usize,
/// PHP files parsed.
pub parsed: usize,
/// Bytes of PHP parsed.
pub parsed_bytes: u64,
/// The walk found more than [`MAX_FILES`] files; the rest are left out.
pub capped: bool,
/// The build was cancelled before it finished.
pub cancelled: bool,
/// Wall time of the walk alone.
pub walk: Duration,
/// Wall time from start to end.
pub elapsed: Duration,
}
/// Stops a running build or updater. Clones share the flag.
#[derive(Debug, Clone, Default)]
pub struct CancelToken(Arc<AtomicBool>);
impl CancelToken {
/// A token that is not cancelled.
pub fn new() -> Self {
Self::default()
}
/// Asks the work to stop at the next file.
pub fn cancel(&self) {
self.0.store(true, Ordering::Relaxed);
}
/// Whether [`CancelToken::cancel`] was called.
pub fn is_cancelled(&self) -> bool {
self.0.load(Ordering::Relaxed)
}
}
/// Worker threads for a build on this machine: one less than the cores,
/// so the UI and the engine keep a core, at least one and at most
/// [`MAX_THREADS`].
pub fn default_threads() -> usize {
std::thread::available_parallelism()
.map(|n| n.get())
.unwrap_or(2)
.saturating_sub(1)
.clamp(1, MAX_THREADS)
}

667
crates/index/src/query.rs Normal file
View file

@ -0,0 +1,667 @@
//! The index in memory and the fuzzy go-to queries over it.
//!
//! Matching uses `nucleo-matcher` (fzf-style scoring: consecutive
//! characters, word starts and path separators score higher). The query is
//! split at whitespace and every word must match; no other syntax is
//! special. Lower-case query characters match either case, upper-case ones
//! only themselves (smart case). Project files and symbols always rank
//! before library ones; within each group the higher score wins, then the
//! shorter name, then the name in byte order, so results are stable.
use std::cmp::Ordering;
use std::collections::BTreeMap;
use nucleo_matcher::pattern::{AtomKind, CaseMatching, Normalization, Pattern};
use nucleo_matcher::{Config, Matcher, Utf32Str};
use crate::{IndexedFile, MAX_FILES, Symbol};
/// Most results one query returns.
pub const MAX_LIMIT: usize = 1_000;
/// Longest query, in characters; the rest is ignored.
pub const MAX_QUERY_CHARS: usize = 256;
/// Score factor of a match inside the file name over a match in the whole
/// path, so `user` finds `User.php` before `user/Repository.php`.
const NAME_WEIGHT: u32 = 2;
/// Files plus symbols from which a query is scanned on several threads.
const PARALLEL_FROM: usize = 20_000;
/// The files and symbols of a project.
#[derive(Debug, Default)]
pub struct Index {
/// Files by project-relative path.
files: BTreeMap<String, IndexedFile>,
/// Symbols in all files.
symbols: usize,
}
/// A file found by [`Index::find_files`].
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct FileHit {
/// Project-relative path with `/` separators.
pub path: String,
/// Match score; higher is better.
pub score: u32,
/// Matched character ranges in `path`, `[start, end)` in UTF-16 code
/// units, ascending and not overlapping.
pub matches: Vec<[usize; 2]>,
/// Library code (below `vendor/`).
pub library: bool,
}
/// A symbol found by [`Index::find_classes`] or [`Index::find_symbols`].
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct SymbolHit<'a> {
/// The symbol.
pub symbol: &'a Symbol,
/// Project-relative path of its file.
pub path: &'a str,
/// Library code (below `vendor/`).
pub library: bool,
/// Match score; higher is better.
pub score: u32,
}
impl Index {
/// An empty index.
pub fn new() -> Self {
Self::default()
}
/// Number of files.
pub fn len(&self) -> usize {
self.files.len()
}
/// Whether the index holds no files.
pub fn is_empty(&self) -> bool {
self.files.is_empty()
}
/// Number of symbols in all files.
pub fn symbol_count(&self) -> usize {
self.symbols
}
/// The file at the project-relative `path`.
pub fn get(&self, path: &str) -> Option<&IndexedFile> {
self.files.get(path)
}
/// Whether the index holds the file at `path`.
pub fn contains(&self, path: &str) -> bool {
self.files.contains_key(path)
}
/// Adds `file`, replacing the entry with the same path. A new file is
/// refused (returns `false`) when the index already holds
/// [`MAX_FILES`] files.
pub fn insert(&mut self, file: IndexedFile) -> bool {
if self.files.len() >= MAX_FILES && !self.files.contains_key(&file.path) {
return false;
}
self.symbols = self.symbols.saturating_add(file.symbols.len());
if let Some(old) = self.files.insert(file.path.clone(), file) {
self.symbols = self.symbols.saturating_sub(old.symbols.len());
}
true
}
/// Drops the file at `path`, or every file below the directory `path`.
/// Returns the number of files dropped.
pub fn remove(&mut self, path: &str) -> usize {
let path = path.trim_matches('/');
if let Some(old) = self.files.remove(path) {
self.symbols = self.symbols.saturating_sub(old.symbols.len());
return 1;
}
let keys: Vec<String> = if path.is_empty() {
self.files.keys().cloned().collect()
} else {
let start = format!("{path}/");
let end = format!("{path}0");
self.files
.range(start..end)
.map(|(k, _)| k.clone())
.collect()
};
for key in &keys {
if let Some(old) = self.files.remove(key) {
self.symbols = self.symbols.saturating_sub(old.symbols.len());
}
}
keys.len()
}
/// Drops everything.
pub fn clear(&mut self) {
self.files.clear();
self.symbols = 0;
}
/// Files whose project-relative path matches `query`, best first, at
/// most `limit` (capped at [`MAX_LIMIT`]).
///
/// A query without `/` (or `\`, read as `/`) is matched against the
/// file name first, which scores [`NAME_WEIGHT`] times higher, and
/// against the whole path otherwise; a query with `/` against the whole
/// path only. An empty query finds nothing.
pub fn find_files(&self, query: &str, limit: usize) -> Vec<FileHit> {
let query = clamp(query).replace('\\', "/");
let by_path = query.contains('/');
let Some(mut fuzzy) = Fuzzy::new(&query, Config::DEFAULT.match_paths()) else {
return Vec::new();
};
let order = |a: &(&IndexedFile, u32, bool), b: &(&IndexedFile, u32, bool)| {
rank((a.0.library, a.1, &a.0.path), (b.0.library, b.1, &b.0.path))
};
let hits = self.scan(
limit,
|| Fuzzy::new(&query, Config::DEFAULT.match_paths()),
|fuzzy, file, hits| {
let name = file_name(&file.path);
if !by_path && let Some(score) = fuzzy.score(name) {
hits.push((file, score.saturating_mul(NAME_WEIGHT), true));
} else if let Some(score) = fuzzy.score(&file.path) {
hits.push((file, score, false));
}
},
order,
);
hits.into_iter()
.map(|(file, score, on_name)| {
let name = file_name(&file.path);
let (haystack, base) = if on_name {
let prefix = file.path.len().saturating_sub(name.len());
let base = file.path.get(..prefix).map_or(0, utf16_len);
(name, base)
} else {
(file.path.as_str(), 0)
};
FileHit {
path: file.path.clone(),
score,
matches: ranges(haystack, &fuzzy.indices(haystack), base),
library: file.library,
}
})
.collect()
}
/// Classes, interfaces, traits and enums whose name matches `query`,
/// best first, at most `limit` (capped at [`MAX_LIMIT`]). A query with
/// `\` is matched against the fully qualified name instead; a leading
/// `\` is ignored. An empty query finds nothing.
pub fn find_classes(&self, query: &str, limit: usize) -> Vec<SymbolHit<'_>> {
self.find_symbols_where(query, limit, true)
}
/// Functions, methods, properties, constants and enum cases whose name
/// matches `query`, best first, at most `limit` (capped at
/// [`MAX_LIMIT`]). `Class::member` matches the container's short name
/// (or its FQN when the part before `::` has a `\`) and the member name
/// separately; `Class::` lists the members of matching classes. A query
/// with `\` and no `::` is matched against the fully qualified name. An
/// empty query finds nothing.
pub fn find_symbols(&self, query: &str, limit: usize) -> Vec<SymbolHit<'_>> {
self.find_symbols_where(query, limit, false)
}
/// The shared part of [`Index::find_classes`] (`classes`) and
/// [`Index::find_symbols`].
fn find_symbols_where<'a>(
&'a self,
query: &str,
limit: usize,
classes: bool,
) -> Vec<SymbolHit<'a>> {
let query = clamp(query);
let query = query.trim().trim_start_matches('\\');
let config = symbol_config();
let split = if classes {
None
} else {
query.split_once("::")
};
let (owner_query, member_query) = match split {
Some((owner, member)) => (Some(owner.trim_start_matches('\\')), member),
None => (None, query),
};
let owner_by_fqn = owner_query.is_some_and(|q| q.contains('\\'));
let by_fqn = owner_query.is_none() && member_query.contains('\\');
let matchers = || {
let owner = owner_query.and_then(|q| Fuzzy::new(q, config.clone()));
let member = Fuzzy::new(member_query, config.clone());
(owner.is_some() || member.is_some()).then_some((owner, member))
};
if matchers().is_none() {
return Vec::new();
}
let visit = |(owner, member): &mut (Option<Fuzzy>, Option<Fuzzy>),
file: &'a IndexedFile,
hits: &mut Vec<SymbolHit<'a>>| {
for symbol in &file.symbols {
if symbol.kind.is_class_like() != classes {
continue;
}
let mut score = 0u32;
if let Some(owner) = owner.as_mut() {
let Some(container) = symbol.container.as_deref() else {
continue;
};
let haystack = if owner_by_fqn {
container
} else {
container.rsplit('\\').next().unwrap_or(container)
};
let Some(s) = owner.score(haystack) else {
continue;
};
score = s;
}
if let Some(member) = member.as_mut() {
let haystack = if by_fqn { &symbol.fqn } else { &symbol.name };
let Some(s) = member.score(haystack) else {
continue;
};
score = score.saturating_add(s);
}
hits.push(SymbolHit {
symbol,
path: &file.path,
library: file.library,
score,
});
}
};
self.scan(limit, matchers, visit, |a, b| {
rank(
(a.library, a.score, &a.symbol.name),
(b.library, b.score, &b.symbol.name),
)
.then_with(|| a.symbol.fqn.cmp(&b.symbol.fqn))
.then_with(|| a.path.cmp(b.path))
.then_with(|| a.symbol.line.cmp(&b.symbol.line))
})
}
/// Runs `visit` over every file and returns the best `limit` hits in
/// `order`. Large indexes are scanned on [`crate::default_threads`]
/// threads, each with its own matcher state from `make` and its own
/// best hits, merged at the end; small ones on the calling thread.
/// Returns nothing when `make` gives no state (an empty query).
fn scan<'a, S, T: Send>(
&'a self,
limit: usize,
make: impl Fn() -> Option<S> + Sync,
visit: impl Fn(&mut S, &'a IndexedFile, &mut Vec<T>) + Sync,
order: impl Fn(&T, &T) -> Ordering + Sync,
) -> Vec<T> {
let files: Vec<&IndexedFile> = self.files.values().collect();
let work = files.len().saturating_add(self.symbols);
let threads = if work < PARALLEL_FROM {
1
} else {
crate::default_threads()
};
let chunk = files.len().div_ceil(threads).max(1);
let run = |part: &[&'a IndexedFile]| {
let mut hits = Vec::new();
if let Some(mut state) = make() {
for file in part {
visit(&mut state, file, &mut hits);
}
}
best(&mut hits, limit, &order);
hits
};
let mut hits: Vec<T> = if threads <= 1 {
run(&files)
} else {
std::thread::scope(|scope| {
let handles: Vec<_> = files
.chunks(chunk)
.map(|part| scope.spawn(|| run(part)))
.collect();
handles
.into_iter()
.flat_map(|h| h.join().unwrap_or_default())
.collect()
})
};
best(&mut hits, limit, &order);
hits
}
}
/// The ranking order: project before library, higher score first, then
/// shorter, then byte order.
fn rank(a: (bool, u32, &str), b: (bool, u32, &str)) -> Ordering {
a.0.cmp(&b.0)
.then_with(|| b.1.cmp(&a.1))
.then_with(|| a.2.len().cmp(&b.2.len()))
.then_with(|| a.2.cmp(b.2))
}
/// Keeps the `limit` (at most [`MAX_LIMIT`]) first items of `hits` in
/// `order`, sorted.
fn best<T>(hits: &mut Vec<T>, limit: usize, order: impl Fn(&T, &T) -> Ordering) {
let limit = limit.min(MAX_LIMIT);
if limit == 0 {
hits.clear();
return;
}
if hits.len() > limit {
hits.select_nth_unstable_by(limit.saturating_sub(1), &order);
hits.truncate(limit);
}
hits.sort_by(order);
}
/// The matcher configuration for symbol names: the default word
/// boundaries (camel case, `\`, `_`, `:` and other punctuation start
/// words), and matches near the start score a little higher.
fn symbol_config() -> Config {
let mut config = Config::DEFAULT;
config.prefer_prefix = true;
config
}
/// The first [`MAX_QUERY_CHARS`] characters of `query`.
fn clamp(query: &str) -> String {
query.chars().take(MAX_QUERY_CHARS).collect()
}
/// The file name of a project-relative path.
fn file_name(path: &str) -> &str {
path.rsplit('/').next().unwrap_or(path)
}
/// Length of `text` in UTF-16 code units.
fn utf16_len(text: &str) -> usize {
text.chars().map(char::len_utf16).sum()
}
/// Turns sorted character indices into `text` into merged `[start, end)`
/// UTF-16 ranges, shifted by `base`.
fn ranges(text: &str, indices: &[u32], base: usize) -> Vec<[usize; 2]> {
let mut out: Vec<[usize; 2]> = Vec::new();
let mut wanted = indices.iter().copied().peekable();
let mut at = base;
for (i, ch) in text.chars().enumerate() {
let width = ch.len_utf16();
if wanted
.peek()
.is_some_and(|&w| usize::try_from(w).is_ok_and(|w| w == i))
{
wanted.next();
match out.last_mut() {
Some(range) if range[1] == at => range[1] = at.saturating_add(width),
_ => out.push([at, at.saturating_add(width)]),
}
}
at = at.saturating_add(width);
}
out
}
/// A compiled query with the matcher and buffers it needs.
struct Fuzzy {
/// The query words.
pattern: Pattern,
/// The matcher and its scratch memory.
matcher: Matcher,
/// Characters of the current haystack, for non-ASCII text.
chars: Vec<char>,
/// Match indices, reused.
indices: Vec<u32>,
}
impl Fuzzy {
/// The query `query` with `config`; `None` for a query without words.
fn new(query: &str, config: Config) -> Option<Self> {
let pattern = Pattern::new(
query,
CaseMatching::Smart,
Normalization::Smart,
AtomKind::Fuzzy,
);
if pattern.atoms.is_empty() {
return None;
}
Some(Self {
pattern,
matcher: Matcher::new(config),
chars: Vec::new(),
indices: Vec::new(),
})
}
/// The score of `haystack`, `None` if it does not match.
fn score(&mut self, haystack: &str) -> Option<u32> {
let Self {
pattern,
matcher,
chars,
..
} = self;
pattern.score(utf32(haystack, chars), matcher)
}
/// The matched character indices in `haystack`, sorted, without
/// duplicates; empty if it does not match.
fn indices(&mut self, haystack: &str) -> Vec<u32> {
let Self {
pattern,
matcher,
chars,
indices,
} = self;
indices.clear();
if pattern
.indices(utf32(haystack, chars), matcher, indices)
.is_none()
{
return Vec::new();
}
indices.sort_unstable();
indices.dedup();
indices.clone()
}
}
/// `text` as the matcher's string type, one entry per Unicode scalar
/// value, so match indices are character indices (`Utf32Str::new` would
/// merge grapheme clusters).
fn utf32<'a>(text: &'a str, chars: &'a mut Vec<char>) -> Utf32Str<'a> {
if text.is_ascii() {
return Utf32Str::Ascii(text.as_bytes());
}
chars.clear();
chars.extend(text.chars());
Utf32Str::Unicode(chars)
}
#[cfg(test)]
mod tests {
use super::*;
use crate::SymbolKind;
fn file(path: &str, symbols: &[(&str, &str, SymbolKind, Option<&str>)]) -> IndexedFile {
IndexedFile {
path: path.into(),
library: crate::is_library(path),
symbols: symbols
.iter()
.map(|(name, fqn, kind, container)| Symbol {
name: (*name).into(),
fqn: (*fqn).into(),
kind: *kind,
container: container.map(str::to_string),
line: 1,
column: 2,
})
.collect(),
}
}
fn sample() -> Index {
let mut index = Index::new();
index.insert(file(
"src/Models/User.php",
&[
("User", "App\\Models\\User", SymbolKind::Class, None),
(
"find",
"App\\Models\\User::find",
SymbolKind::Method,
Some("App\\Models\\User"),
),
(
"$name",
"App\\Models\\User::$name",
SymbolKind::Property,
Some("App\\Models\\User"),
),
],
));
index.insert(file(
"src/user/Repository.php",
&[(
"Repository",
"App\\User\\Repository",
SymbolKind::Class,
None,
)],
));
index.insert(file(
"vendor/acme/lib/src/User.php",
&[
("User", "Acme\\User", SymbolKind::Class, None),
(
"find",
"Acme\\User::find",
SymbolKind::Method,
Some("Acme\\User"),
),
],
));
index.insert(file("templates/user.html.twig", &[]));
index.insert(file(
"src/helpers.php",
&[("format_user", "format_user", SymbolKind::Function, None)],
));
index
}
#[test]
fn counts_and_removal() {
let mut index = sample();
assert_eq!(index.len(), 5);
assert_eq!(index.symbol_count(), 7);
index.insert(file("src/helpers.php", &[]));
assert_eq!(index.symbol_count(), 6);
assert_eq!(index.remove("src/user"), 1);
assert_eq!(index.remove("src/Models/User.php"), 1);
assert_eq!(index.remove("src/Models"), 0);
assert_eq!(index.symbol_count(), 2);
assert!(index.contains("src/helpers.php"));
assert_eq!(index.remove(""), 3);
assert!(index.is_empty());
assert_eq!(index.symbol_count(), 0);
}
#[test]
fn prefix_removal_respects_segments() {
let mut index = Index::new();
index.insert(file("src/a.php", &[]));
index.insert(file("src2/b.php", &[]));
index.insert(file("src-x/c.php", &[]));
assert_eq!(index.remove("src"), 1);
assert_eq!(index.len(), 2);
}
#[test]
fn files_rank_names_then_paths_and_project_first() {
let index = sample();
let paths: Vec<String> = index
.find_files("user", 10)
.into_iter()
.map(|h| h.path)
.collect();
assert_eq!(
paths,
vec![
"src/Models/User.php",
"templates/user.html.twig",
"src/user/Repository.php",
"vendor/acme/lib/src/User.php",
]
);
let hit = &index.find_files("user", 1)[0];
assert_eq!(hit.matches, vec![[11, 15]]);
let by_path = index.find_files("models/u", 10);
assert_eq!(by_path[0].path, "src/Models/User.php");
assert!(index.find_files(" ", 10).is_empty());
assert!(index.find_files("user", 0).is_empty());
assert!(index.find_files("zzz", 10).is_empty());
}
#[test]
fn classes_and_symbols() {
let index = sample();
let classes: Vec<(&str, bool)> = index
.find_classes("user", 10)
.iter()
.map(|h| (h.symbol.fqn.as_str(), h.library))
.collect();
assert_eq!(
classes,
vec![("App\\Models\\User", false), ("Acme\\User", true)]
);
let fqn = index.find_classes("\\acme\\us", 10);
assert_eq!(fqn.len(), 1);
assert_eq!(fqn[0].symbol.fqn, "Acme\\User");
let finds: Vec<&str> = index
.find_symbols("find", 10)
.iter()
.map(|h| h.symbol.fqn.as_str())
.collect();
assert_eq!(finds, vec!["App\\Models\\User::find", "Acme\\User::find"]);
let members: Vec<&str> = index
.find_symbols("User::", 10)
.iter()
.map(|h| h.symbol.name.as_str())
.collect();
assert_eq!(members, vec!["find", "$name", "find"]);
let scoped = index.find_symbols("acme\\user::f", 10);
assert_eq!(scoped.len(), 1);
assert!(
index
.find_symbols("user", 10)
.iter()
.all(|h| !h.symbol.kind.is_class_like())
);
assert!(index.find_symbols("::", 10).is_empty());
}
#[test]
fn match_ranges_count_utf16() {
assert_eq!(ranges("aéb", &[0, 1, 2], 0), vec![[0, 3]]);
assert_eq!(ranges("a😀b", &[2], 4), vec![[7, 8]]);
assert_eq!(ranges("abc", &[0, 2], 0), vec![[0, 1], [2, 3]]);
}
#[test]
fn caps_the_number_of_files() {
let mut index = Index::new();
for i in 0..MAX_FILES {
index.files.insert(format!("f{i}"), IndexedFile::default());
}
assert!(!index.insert(file("new.php", &[])));
assert!(index.insert(file("f1", &[])));
}
}

313
crates/index/src/symbol.rs Normal file
View file

@ -0,0 +1,313 @@
//! Symbols and indexed files, extracted from tree-sitter outlines.
use lutra_syntax::{OutlineItem, OutlineKind, Parser};
use crate::{MAX_PARSE_BYTES, MAX_SYMBOLS_PER_FILE};
/// What a symbol declares.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub enum SymbolKind {
/// A class.
Class,
/// An interface.
Interface,
/// A trait.
Trait,
/// An enum.
Enum,
/// A function outside a class.
Function,
/// A method.
Method,
/// A property, declared or constructor-promoted.
Property,
/// A class constant or a `const` outside a class.
Constant,
/// A `case` of an enum.
EnumCase,
}
impl SymbolKind {
/// The kind's name as the engine and the UI use it, the same as the
/// outline's: `class`, `interface`, `trait`, `enum`, `function`,
/// `method`, `property`, `constant` or `enumCase`.
pub fn as_str(self) -> &'static str {
match self {
Self::Class => "class",
Self::Interface => "interface",
Self::Trait => "trait",
Self::Enum => "enum",
Self::Function => "function",
Self::Method => "method",
Self::Property => "property",
Self::Constant => "constant",
Self::EnumCase => "enumCase",
}
}
/// Whether the kind is a class, interface, trait or enum (Go to Class);
/// the others are found by Go to Symbol.
pub fn is_class_like(self) -> bool {
matches!(
self,
Self::Class | Self::Interface | Self::Trait | Self::Enum
)
}
/// The symbol kind of an outline kind; `None` for a namespace.
fn from_outline(kind: OutlineKind) -> Option<Self> {
Some(match kind {
OutlineKind::Namespace => return None,
OutlineKind::Class => Self::Class,
OutlineKind::Interface => Self::Interface,
OutlineKind::Trait => Self::Trait,
OutlineKind::Enum => Self::Enum,
OutlineKind::Function => Self::Function,
OutlineKind::Method => Self::Method,
OutlineKind::Property => Self::Property,
OutlineKind::Constant => Self::Constant,
OutlineKind::EnumCase => Self::EnumCase,
})
}
}
/// One declaration.
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
pub struct Symbol {
/// The name as written: `Product`, `find`, `$id` (properties keep the
/// `$`), `MAX`.
pub name: String,
/// The fully qualified name without a leading `\`: `App\Models\Product`
/// for a class or function, `App\Models\Product::find` for a member.
pub fqn: String,
/// What it declares.
pub kind: SymbolKind,
/// The FQN of the class, interface, trait or enum that declares a
/// member; `None` for classes, functions and constants outside a class.
pub container: Option<String>,
/// 0-based line of the name.
pub line: usize,
/// Column of the name in UTF-16 code units.
pub column: usize,
}
/// A file in the index.
#[derive(Debug, Clone, PartialEq, Eq, Default)]
pub struct IndexedFile {
/// Project-relative path with `/` separators.
pub path: String,
/// Library code: the file is below the root's `vendor/`.
pub library: bool,
/// Declarations in source order; empty for files that are not parsed.
pub symbols: Vec<Symbol>,
}
/// Whether the project-relative `path` is library code (the root's
/// `vendor/` or anything below it).
pub fn is_library(path: &str) -> bool {
path == "vendor" || path.starts_with("vendor/")
}
/// Whether the project-relative `path` is a PHP file to parse.
pub(crate) fn is_php(path: &str) -> bool {
path.rsplit('/')
.next()
.and_then(|name| name.rsplit_once('.'))
.is_some_and(|(stem, ext)| !stem.is_empty() && ext.eq_ignore_ascii_case("php"))
}
/// Indexes `text` as the file at `path`: parses it when it is a PHP file
/// of at most [`MAX_PARSE_BYTES`] and extracts its symbols; other files get
/// no symbols.
///
/// # Parameters
/// - `parser`: reused between files.
/// - `path`: project-relative path with `/` separators.
/// - `text`: the file's text.
pub fn index_text(parser: &mut Parser, path: String, text: &str) -> IndexedFile {
let library = is_library(&path);
let parse = is_php(&path) && text.len() as u64 <= MAX_PARSE_BYTES;
let symbols = if parse {
let tree = parser.parse(text);
symbols_from_outline(&lutra_syntax::outline(&tree))
} else {
Vec::new()
};
IndexedFile {
path,
library,
symbols,
}
}
/// The symbols declared in an outline, in source order, at most
/// [`MAX_SYMBOLS_PER_FILE`].
///
/// Namespaces give the FQN prefix of what they contain; classes,
/// interfaces, traits and enums are the container of their members.
/// Anonymous classes and their members are skipped: nothing can navigate to
/// them by name.
pub fn symbols_from_outline(items: &[OutlineItem]) -> Vec<Symbol> {
let mut out = Vec::new();
collect(items, "", None, &mut out);
out
}
/// Adds the symbols of `items` to `out`.
///
/// # Parameters
/// - `namespace`: the enclosing namespace without leading or trailing `\`;
/// empty for the global namespace.
/// - `container`: the FQN of the enclosing class-like declaration.
fn collect(items: &[OutlineItem], namespace: &str, container: Option<&str>, out: &mut Vec<Symbol>) {
for item in items {
if out.len() >= MAX_SYMBOLS_PER_FILE {
return;
}
let Some(kind) = SymbolKind::from_outline(item.kind) else {
let name = item.name.trim_matches('\\');
collect(&item.children, name, None, out);
continue;
};
if item.is_anonymous() || item.name.is_empty() {
continue;
}
let (fqn, owner) = match container {
Some(owner) if !kind.is_class_like() && kind != SymbolKind::Function => {
(format!("{owner}::{}", item.name), Some(owner.to_string()))
}
_ => (qualify(namespace, &item.name), None),
};
let start = item.selection_range.start;
out.push(Symbol {
name: item.name.clone(),
fqn: fqn.clone(),
kind,
container: owner,
line: start.line,
column: start.column,
});
if kind.is_class_like() {
collect(&item.children, namespace, Some(&fqn), out);
}
}
}
/// `name` qualified with `namespace`: `App\Product`, or `Product` in the
/// global namespace.
fn qualify(namespace: &str, name: &str) -> String {
let name = name.trim_start_matches('\\');
if namespace.is_empty() {
name.to_string()
} else {
format!("{namespace}\\{name}")
}
}
#[cfg(test)]
mod tests {
use super::*;
fn symbols(source: &str) -> Vec<Symbol> {
let mut parser = Parser::new().unwrap();
index_text(&mut parser, "src/a.php".into(), source).symbols
}
#[test]
fn qualifies_classes_members_and_functions() {
let found = symbols(
"<?php\nnamespace App\\Models;\n\ninterface HasId {}\nfinal class Product implements HasId\n{\n public const MAX = 3;\n private ?string $name = null;\n public function __construct(private int $id) {}\n public function find(int $id): ?self { return null; }\n}\nenum Status: string { case Draft = 'draft'; }\nfunction helper(): void {}\nconst VERSION = 1;\n$x = new class { public function hidden() {} };\n",
);
let summary: Vec<(String, &str, Option<String>)> = found
.iter()
.map(|s| (s.fqn.clone(), s.kind.as_str(), s.container.clone()))
.collect();
let product = Some("App\\Models\\Product".to_string());
assert_eq!(
summary,
vec![
("App\\Models\\HasId".into(), "interface", None),
("App\\Models\\Product".into(), "class", None),
(
"App\\Models\\Product::MAX".into(),
"constant",
product.clone()
),
(
"App\\Models\\Product::$name".into(),
"property",
product.clone()
),
(
"App\\Models\\Product::__construct".into(),
"method",
product.clone()
),
(
"App\\Models\\Product::$id".into(),
"property",
product.clone()
),
("App\\Models\\Product::find".into(), "method", product),
("App\\Models\\Status".into(), "enum", None),
(
"App\\Models\\Status::Draft".into(),
"enumCase",
Some("App\\Models\\Status".into())
),
("App\\Models\\helper".into(), "function", None),
("App\\Models\\VERSION".into(), "constant", None),
]
);
let product = found.iter().find(|s| s.name == "Product").unwrap();
assert_eq!((product.line, product.column), (4, 12));
}
#[test]
fn braced_and_global_namespaces() {
let found = symbols(
"<?php\nnamespace A { class One {} }\nnamespace { class Two {} function three() {} }\n",
);
let fqns: Vec<&str> = found.iter().map(|s| s.fqn.as_str()).collect();
assert_eq!(fqns, vec!["A\\One", "Two", "three"]);
}
#[test]
fn only_php_files_are_parsed() {
let mut parser = Parser::new().unwrap();
let text = "<?php class A {}";
assert_eq!(
index_text(&mut parser, "a.php".into(), text).symbols.len(),
1
);
assert_eq!(
index_text(&mut parser, "A.PHP".into(), text).symbols.len(),
1
);
assert!(
index_text(&mut parser, "a.twig".into(), text)
.symbols
.is_empty()
);
assert!(
index_text(&mut parser, ".php".into(), text)
.symbols
.is_empty()
);
let file = index_text(&mut parser, "vendor/x/a.php".into(), text);
assert!(file.library);
assert!(!is_library("vendors/a.php"));
}
#[test]
fn oversized_files_have_no_symbols() {
let mut parser = Parser::new().unwrap();
let mut text = String::from("<?php class A {}\n");
text.push_str(&"// filler\n".repeat((MAX_PARSE_BYTES as usize) / 10 + 1));
assert!(
index_text(&mut parser, "a.php".into(), &text)
.symbols
.is_empty()
);
}
}

192
crates/index/src/updater.rs Normal file
View file

@ -0,0 +1,192 @@
//! Incremental updates: re-indexing single files and directories after the
//! build, on a background thread.
use std::collections::HashMap;
use std::path::PathBuf;
use std::sync::mpsc::{Receiver, Sender, channel};
use lutra_syntax::Parser;
use lutra_text::Buffer;
use crate::build::{guarded_index, index_file};
use crate::walk::walk;
use crate::{CancelToken, IndexedFile, MAX_FILES, Update};
/// Most files one directory rescan reports in one [`Update::Files`].
const RESCAN_BATCH: usize = 256;
/// Work for an [`Updater`].
#[derive(Debug, Clone)]
pub enum Job {
/// Re-index one file.
File {
/// Project-relative path with `/` separators.
path: String,
/// The text of an open editor buffer to index instead of the file on
/// disk; `None` reads the disk, and reports [`Update::Removed`] when
/// the file is gone or now excluded by the project's rules.
text: Option<Buffer>,
},
/// Re-index a directory: drop everything below it, then list it again
/// from disk.
Dir {
/// Project-relative path with `/` separators.
path: String,
},
}
impl Job {
/// The path the job is about.
fn path(&self) -> &str {
match self {
Self::File { path, .. } | Self::Dir { path } => path,
}
}
}
/// A background thread that runs [`Job`]s one batch at a time and sends
/// the results as live [`Update`]s. Jobs queued while it works are
/// coalesced: only the last job per path runs, so a burst of keystrokes in
/// one buffer parses it once. The thread ends when the updater is dropped
/// or `cancel` fires.
#[derive(Debug)]
pub struct Updater {
/// Queues jobs for the thread.
jobs: Sender<Job>,
}
impl Updater {
/// Starts the thread for the project at `root`.
///
/// # Parameters
/// - `root`: the project root (absolute).
/// - `cancel`: stops the thread between jobs.
/// - `updates`: where results go, usually the build's channel.
///
/// # Errors
/// The thread could not be started.
pub fn spawn(
root: PathBuf,
cancel: CancelToken,
updates: Sender<Update>,
) -> std::io::Result<Self> {
let (jobs, queue) = channel();
std::thread::Builder::new()
.name("lutra-index-update".into())
.spawn(move || run(&root, &cancel, &queue, &updates))?;
Ok(Self { jobs })
}
/// Queues `job`. Returns `false` if the thread has ended.
pub fn send(&self, job: Job) -> bool {
self.jobs.send(job).is_ok()
}
}
/// The updater thread's loop.
fn run(
root: &std::path::Path,
cancel: &CancelToken,
queue: &Receiver<Job>,
updates: &Sender<Update>,
) {
let mut parser = Parser::new().ok();
while let Ok(first) = queue.recv() {
let mut jobs = vec![first];
jobs.extend(queue.try_iter());
for job in coalesce(jobs) {
if cancel.is_cancelled() {
return;
}
let sent = match job {
Job::File {
path,
text: Some(text),
} => {
let library = crate::is_library(&path);
let file = guarded_index(&mut parser, path.clone(), &text.text()).unwrap_or(
IndexedFile {
path,
library,
symbols: Vec::new(),
},
);
updates
.send(Update::Files {
files: vec![file],
live: true,
})
.is_ok()
}
Job::File { path, text: None } => {
rescan(root, &path, false, cancel, &mut parser, updates)
}
Job::Dir { path } => rescan(root, &path, true, cancel, &mut parser, updates),
};
if !sent {
return;
}
}
}
}
/// Keeps the last job per path, in the order of those last jobs.
fn coalesce(jobs: Vec<Job>) -> Vec<Job> {
let mut last: HashMap<String, usize> = HashMap::new();
for (i, job) in jobs.iter().enumerate() {
last.insert(job.path().to_string(), i);
}
jobs.into_iter()
.enumerate()
.filter(|(i, job)| last.get(job.path()) == Some(i))
.map(|(_, job)| job)
.collect()
}
/// Lists `path` from disk with the project's rules and indexes what is
/// found. A file that is no longer found is reported removed; a directory
/// is always reported removed first, so files gone from it disappear.
/// Returns `false` once the receiver is gone.
fn rescan(
root: &std::path::Path,
path: &str,
dir: bool,
cancel: &CancelToken,
parser: &mut Option<Parser>,
updates: &Sender<Update>,
) -> bool {
let walked = walk(root, Some(path), 1, MAX_FILES, cancel);
if (dir || walked.files.is_empty())
&& updates
.send(Update::Removed {
path: path.to_string(),
})
.is_err()
{
return false;
}
let mut batch = Vec::new();
for found in &walked.files {
if cancel.is_cancelled() {
return true;
}
batch.push(index_file(parser, found).0);
if batch.len() >= RESCAN_BATCH
&& updates
.send(Update::Files {
files: std::mem::take(&mut batch),
live: true,
})
.is_err()
{
return false;
}
}
batch.is_empty()
|| updates
.send(Update::Files {
files: batch,
live: true,
})
.is_ok()
}

193
crates/index/src/walk.rs Normal file
View file

@ -0,0 +1,193 @@
//! The walk over the project: which files the index lists (see the crate
//! docs for the rules).
use std::path::{Path, PathBuf};
use std::sync::Mutex;
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
use ignore::{DirEntry, WalkBuilder, WalkState};
use crate::CancelToken;
use crate::symbol::is_php;
/// A file the walk found.
#[derive(Debug, Clone, PartialEq, Eq)]
pub(crate) struct Found {
/// Project-relative path with `/` separators.
pub path: String,
/// Absolute path.
pub abs: PathBuf,
/// Size in bytes for PHP files; 0 for other files, which are not read.
pub size: u64,
}
/// What a walk returns.
#[derive(Debug, Default)]
pub(crate) struct Walked {
/// The files, in no particular order.
pub files: Vec<Found>,
/// More than the limit were found; the rest are left out.
pub capped: bool,
}
/// Lists the files below `root`, or only those at or below the
/// project-relative `scope` (a file or a directory), with the project's
/// rules.
///
/// # Parameters
/// - `root`: the project root (absolute).
/// - `scope`: `None` for the whole project.
/// - `threads`: walker threads.
/// - `limit`: most files to return.
/// - `cancel`: stops the walk.
///
/// Entries that cannot be read are skipped, as the project tree skips
/// them.
pub(crate) fn walk(
root: &Path,
scope: Option<&str>,
threads: usize,
limit: usize,
cancel: &CancelToken,
) -> Walked {
let found = Mutex::new(Vec::new());
let count = AtomicUsize::new(0);
let capped = AtomicBool::new(false);
let scope = scope.map(|s| s.trim_matches('/').to_string());
walker(root, scope.clone(), threads)
.build_parallel()
.run(|| {
let found = &found;
let count = &count;
let capped = &capped;
let scope = scope.as_deref();
Box::new(move |entry| {
if cancel.is_cancelled() || capped.load(Ordering::Relaxed) {
return WalkState::Quit;
}
let Ok(entry) = entry else {
return WalkState::Continue;
};
if !entry.file_type().is_some_and(|t| t.is_file()) {
return WalkState::Continue;
}
let Some(rel) = relative(root, entry.path()) else {
return WalkState::Continue;
};
if scope.is_some_and(|s| !inside(&rel, s)) {
return WalkState::Continue;
}
if count.fetch_add(1, Ordering::Relaxed) >= limit {
capped.store(true, Ordering::Relaxed);
return WalkState::Quit;
}
let size = if is_php(&rel) {
entry.metadata().map(|m| m.len()).unwrap_or(0)
} else {
0
};
if let Ok(mut files) = found.lock() {
files.push(Found {
path: rel,
abs: entry.path().to_path_buf(),
size,
});
}
WalkState::Continue
})
});
Walked {
files: found.into_inner().unwrap_or_default(),
capped: capped.load(Ordering::Relaxed),
}
}
/// The walker over `root` with the project's rules, entering only
/// directories on the way to or inside `scope`.
fn walker(root: &Path, scope: Option<String>, threads: usize) -> WalkBuilder {
let mut builder = WalkBuilder::new(root);
builder
.hidden(false)
.parents(false)
.ignore(true)
.git_ignore(true)
.git_exclude(true)
.git_global(true)
.require_git(false)
.follow_links(false)
.threads(threads.max(1));
let vendor = root.join("vendor");
let vendor_is_dir = std::fs::symlink_metadata(&vendor).is_ok_and(|m| m.is_dir());
let vendor_in_scope = scope
.as_deref()
.is_none_or(|s| related("vendor", s) || s.is_empty());
if vendor_is_dir && vendor_in_scope {
builder.add(&vendor);
}
let root = root.to_path_buf();
builder.filter_entry(move |entry| keep_entry(&root, &vendor, scope.as_deref(), entry));
builder
}
/// Whether the walk enters (directory) or visits (file) `entry`: `.git`
/// and `node_modules` are skipped, the root's `vendor` is reached through
/// its own walk root, and with a scope only directories on the way to or
/// inside it are entered.
fn keep_entry(root: &Path, vendor: &Path, scope: Option<&str>, entry: &DirEntry) -> bool {
let name = entry.file_name();
if name == ".git" {
return false;
}
let is_dir = entry.file_type().is_some_and(|t| t.is_dir());
if is_dir && name == "node_modules" {
return false;
}
if entry.depth() == 1 && entry.path() == vendor {
return false;
}
if is_dir
&& let Some(scope) = scope
&& let Some(rel) = relative(root, entry.path())
{
return related(&rel, scope);
}
true
}
/// Whether `path` is `scope` or below it.
fn inside(path: &str, scope: &str) -> bool {
scope.is_empty()
|| path == scope
|| path
.strip_prefix(scope)
.is_some_and(|rest| rest.starts_with('/'))
}
/// Whether the directory `dir` is on the way to `scope` or inside it.
fn related(dir: &str, scope: &str) -> bool {
inside(dir, scope) || inside(scope, dir)
}
/// `path` relative to `root` with `/` separators; `None` outside the root
/// or for a name that is not valid Unicode.
fn relative(root: &Path, path: &Path) -> Option<String> {
let rel = path.strip_prefix(root).ok()?;
let parts: Option<Vec<&str>> = rel.components().map(|c| c.as_os_str().to_str()).collect();
Some(parts?.join("/"))
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn scope_relations() {
assert!(inside("src/a.php", "src"));
assert!(inside("src", "src"));
assert!(!inside("src2/a.php", "src"));
assert!(inside("anything", ""));
assert!(related("src", "src/deep/a.php"));
assert!(related("src/deep/x", "src/deep"));
assert!(!related("lib", "src/a.php"));
}
}