Lutra/crates/index/src/walk.rs
danny 053441d066 M2.5: symbol index and go-to commands
crates/index: walks the project with the search/VFS rules (vendor as
library), parses .php files from tree-sitter outlines on a worker pool
off the engine lock, updates incrementally from disk and buffers through
an updater thread, caps at 200,000 files, and answers fuzzy file, class
and symbol queries (nucleo-matcher, MPL-2.0 as a separate crate).

Engine: index.status, goto.file, goto.class, goto.symbol; events
indexProgress and indexReady drained by the pump; the index starts on
project.open and is cancelled on close. Tests: unit, incremental,
property (random queries and sources), engine and TCP control tests,
release timing on a generated 20k-file project (3.9-4.2 s).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-17 12:53:33 +02:00

193 lines
6.1 KiB
Rust

//! The walk over the project: which files the index lists (see the crate
//! docs for the rules).
use std::path::{Path, PathBuf};
use std::sync::Mutex;
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
use ignore::{DirEntry, WalkBuilder, WalkState};
use crate::CancelToken;
use crate::symbol::is_php;
/// A file the walk found.
#[derive(Debug, Clone, PartialEq, Eq)]
pub(crate) struct Found {
/// Project-relative path with `/` separators.
pub path: String,
/// Absolute path.
pub abs: PathBuf,
/// Size in bytes for PHP files; 0 for other files, which are not read.
pub size: u64,
}
/// What a walk returns.
#[derive(Debug, Default)]
pub(crate) struct Walked {
/// The files, in no particular order.
pub files: Vec<Found>,
/// More than the limit were found; the rest are left out.
pub capped: bool,
}
/// Lists the files below `root`, or only those at or below the
/// project-relative `scope` (a file or a directory), with the project's
/// rules.
///
/// # Parameters
/// - `root`: the project root (absolute).
/// - `scope`: `None` for the whole project.
/// - `threads`: walker threads.
/// - `limit`: most files to return.
/// - `cancel`: stops the walk.
///
/// Entries that cannot be read are skipped, as the project tree skips
/// them.
pub(crate) fn walk(
root: &Path,
scope: Option<&str>,
threads: usize,
limit: usize,
cancel: &CancelToken,
) -> Walked {
let found = Mutex::new(Vec::new());
let count = AtomicUsize::new(0);
let capped = AtomicBool::new(false);
let scope = scope.map(|s| s.trim_matches('/').to_string());
walker(root, scope.clone(), threads)
.build_parallel()
.run(|| {
let found = &found;
let count = &count;
let capped = &capped;
let scope = scope.as_deref();
Box::new(move |entry| {
if cancel.is_cancelled() || capped.load(Ordering::Relaxed) {
return WalkState::Quit;
}
let Ok(entry) = entry else {
return WalkState::Continue;
};
if !entry.file_type().is_some_and(|t| t.is_file()) {
return WalkState::Continue;
}
let Some(rel) = relative(root, entry.path()) else {
return WalkState::Continue;
};
if scope.is_some_and(|s| !inside(&rel, s)) {
return WalkState::Continue;
}
if count.fetch_add(1, Ordering::Relaxed) >= limit {
capped.store(true, Ordering::Relaxed);
return WalkState::Quit;
}
let size = if is_php(&rel) {
entry.metadata().map(|m| m.len()).unwrap_or(0)
} else {
0
};
if let Ok(mut files) = found.lock() {
files.push(Found {
path: rel,
abs: entry.path().to_path_buf(),
size,
});
}
WalkState::Continue
})
});
Walked {
files: found.into_inner().unwrap_or_default(),
capped: capped.load(Ordering::Relaxed),
}
}
/// The walker over `root` with the project's rules, entering only
/// directories on the way to or inside `scope`.
fn walker(root: &Path, scope: Option<String>, threads: usize) -> WalkBuilder {
let mut builder = WalkBuilder::new(root);
builder
.hidden(false)
.parents(false)
.ignore(true)
.git_ignore(true)
.git_exclude(true)
.git_global(true)
.require_git(false)
.follow_links(false)
.threads(threads.max(1));
let vendor = root.join("vendor");
let vendor_is_dir = std::fs::symlink_metadata(&vendor).is_ok_and(|m| m.is_dir());
let vendor_in_scope = scope
.as_deref()
.is_none_or(|s| related("vendor", s) || s.is_empty());
if vendor_is_dir && vendor_in_scope {
builder.add(&vendor);
}
let root = root.to_path_buf();
builder.filter_entry(move |entry| keep_entry(&root, &vendor, scope.as_deref(), entry));
builder
}
/// Whether the walk enters (directory) or visits (file) `entry`: `.git`
/// and `node_modules` are skipped, the root's `vendor` is reached through
/// its own walk root, and with a scope only directories on the way to or
/// inside it are entered.
fn keep_entry(root: &Path, vendor: &Path, scope: Option<&str>, entry: &DirEntry) -> bool {
let name = entry.file_name();
if name == ".git" {
return false;
}
let is_dir = entry.file_type().is_some_and(|t| t.is_dir());
if is_dir && name == "node_modules" {
return false;
}
if entry.depth() == 1 && entry.path() == vendor {
return false;
}
if is_dir
&& let Some(scope) = scope
&& let Some(rel) = relative(root, entry.path())
{
return related(&rel, scope);
}
true
}
/// Whether `path` is `scope` or below it.
fn inside(path: &str, scope: &str) -> bool {
scope.is_empty()
|| path == scope
|| path
.strip_prefix(scope)
.is_some_and(|rest| rest.starts_with('/'))
}
/// Whether the directory `dir` is on the way to `scope` or inside it.
fn related(dir: &str, scope: &str) -> bool {
inside(dir, scope) || inside(scope, dir)
}
/// `path` relative to `root` with `/` separators; `None` outside the root
/// or for a name that is not valid Unicode.
fn relative(root: &Path, path: &Path) -> Option<String> {
let rel = path.strip_prefix(root).ok()?;
let parts: Option<Vec<&str>> = rel.components().map(|c| c.as_os_str().to_str()).collect();
Some(parts?.join("/"))
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn scope_relations() {
assert!(inside("src/a.php", "src"));
assert!(inside("src", "src"));
assert!(!inside("src2/a.php", "src"));
assert!(inside("anything", ""));
assert!(related("src", "src/deep/a.php"));
assert!(related("src/deep/x", "src/deep"));
assert!(!related("lib", "src/a.php"));
}
}