crates/index: walks the project with the search/VFS rules (vendor as library), parses .php files from tree-sitter outlines on a worker pool off the engine lock, updates incrementally from disk and buffers through an updater thread, caps at 200,000 files, and answers fuzzy file, class and symbol queries (nucleo-matcher, MPL-2.0 as a separate crate). Engine: index.status, goto.file, goto.class, goto.symbol; events indexProgress and indexReady drained by the pump; the index starts on project.open and is cancelled on close. Tests: unit, incremental, property (random queries and sources), engine and TCP control tests, release timing on a generated 20k-file project (3.9-4.2 s). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
193 lines
6.1 KiB
Rust
193 lines
6.1 KiB
Rust
//! The walk over the project: which files the index lists (see the crate
|
|
//! docs for the rules).
|
|
|
|
use std::path::{Path, PathBuf};
|
|
use std::sync::Mutex;
|
|
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
|
|
|
|
use ignore::{DirEntry, WalkBuilder, WalkState};
|
|
|
|
use crate::CancelToken;
|
|
use crate::symbol::is_php;
|
|
|
|
/// A file the walk found.
|
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
|
pub(crate) struct Found {
|
|
/// Project-relative path with `/` separators.
|
|
pub path: String,
|
|
/// Absolute path.
|
|
pub abs: PathBuf,
|
|
/// Size in bytes for PHP files; 0 for other files, which are not read.
|
|
pub size: u64,
|
|
}
|
|
|
|
/// What a walk returns.
|
|
#[derive(Debug, Default)]
|
|
pub(crate) struct Walked {
|
|
/// The files, in no particular order.
|
|
pub files: Vec<Found>,
|
|
/// More than the limit were found; the rest are left out.
|
|
pub capped: bool,
|
|
}
|
|
|
|
/// Lists the files below `root`, or only those at or below the
|
|
/// project-relative `scope` (a file or a directory), with the project's
|
|
/// rules.
|
|
///
|
|
/// # Parameters
|
|
/// - `root`: the project root (absolute).
|
|
/// - `scope`: `None` for the whole project.
|
|
/// - `threads`: walker threads.
|
|
/// - `limit`: most files to return.
|
|
/// - `cancel`: stops the walk.
|
|
///
|
|
/// Entries that cannot be read are skipped, as the project tree skips
|
|
/// them.
|
|
pub(crate) fn walk(
|
|
root: &Path,
|
|
scope: Option<&str>,
|
|
threads: usize,
|
|
limit: usize,
|
|
cancel: &CancelToken,
|
|
) -> Walked {
|
|
let found = Mutex::new(Vec::new());
|
|
let count = AtomicUsize::new(0);
|
|
let capped = AtomicBool::new(false);
|
|
let scope = scope.map(|s| s.trim_matches('/').to_string());
|
|
walker(root, scope.clone(), threads)
|
|
.build_parallel()
|
|
.run(|| {
|
|
let found = &found;
|
|
let count = &count;
|
|
let capped = &capped;
|
|
let scope = scope.as_deref();
|
|
Box::new(move |entry| {
|
|
if cancel.is_cancelled() || capped.load(Ordering::Relaxed) {
|
|
return WalkState::Quit;
|
|
}
|
|
let Ok(entry) = entry else {
|
|
return WalkState::Continue;
|
|
};
|
|
if !entry.file_type().is_some_and(|t| t.is_file()) {
|
|
return WalkState::Continue;
|
|
}
|
|
let Some(rel) = relative(root, entry.path()) else {
|
|
return WalkState::Continue;
|
|
};
|
|
if scope.is_some_and(|s| !inside(&rel, s)) {
|
|
return WalkState::Continue;
|
|
}
|
|
if count.fetch_add(1, Ordering::Relaxed) >= limit {
|
|
capped.store(true, Ordering::Relaxed);
|
|
return WalkState::Quit;
|
|
}
|
|
let size = if is_php(&rel) {
|
|
entry.metadata().map(|m| m.len()).unwrap_or(0)
|
|
} else {
|
|
0
|
|
};
|
|
if let Ok(mut files) = found.lock() {
|
|
files.push(Found {
|
|
path: rel,
|
|
abs: entry.path().to_path_buf(),
|
|
size,
|
|
});
|
|
}
|
|
WalkState::Continue
|
|
})
|
|
});
|
|
Walked {
|
|
files: found.into_inner().unwrap_or_default(),
|
|
capped: capped.load(Ordering::Relaxed),
|
|
}
|
|
}
|
|
|
|
/// The walker over `root` with the project's rules, entering only
|
|
/// directories on the way to or inside `scope`.
|
|
fn walker(root: &Path, scope: Option<String>, threads: usize) -> WalkBuilder {
|
|
let mut builder = WalkBuilder::new(root);
|
|
builder
|
|
.hidden(false)
|
|
.parents(false)
|
|
.ignore(true)
|
|
.git_ignore(true)
|
|
.git_exclude(true)
|
|
.git_global(true)
|
|
.require_git(false)
|
|
.follow_links(false)
|
|
.threads(threads.max(1));
|
|
let vendor = root.join("vendor");
|
|
let vendor_is_dir = std::fs::symlink_metadata(&vendor).is_ok_and(|m| m.is_dir());
|
|
let vendor_in_scope = scope
|
|
.as_deref()
|
|
.is_none_or(|s| related("vendor", s) || s.is_empty());
|
|
if vendor_is_dir && vendor_in_scope {
|
|
builder.add(&vendor);
|
|
}
|
|
let root = root.to_path_buf();
|
|
builder.filter_entry(move |entry| keep_entry(&root, &vendor, scope.as_deref(), entry));
|
|
builder
|
|
}
|
|
|
|
/// Whether the walk enters (directory) or visits (file) `entry`: `.git`
|
|
/// and `node_modules` are skipped, the root's `vendor` is reached through
|
|
/// its own walk root, and with a scope only directories on the way to or
|
|
/// inside it are entered.
|
|
fn keep_entry(root: &Path, vendor: &Path, scope: Option<&str>, entry: &DirEntry) -> bool {
|
|
let name = entry.file_name();
|
|
if name == ".git" {
|
|
return false;
|
|
}
|
|
let is_dir = entry.file_type().is_some_and(|t| t.is_dir());
|
|
if is_dir && name == "node_modules" {
|
|
return false;
|
|
}
|
|
if entry.depth() == 1 && entry.path() == vendor {
|
|
return false;
|
|
}
|
|
if is_dir
|
|
&& let Some(scope) = scope
|
|
&& let Some(rel) = relative(root, entry.path())
|
|
{
|
|
return related(&rel, scope);
|
|
}
|
|
true
|
|
}
|
|
|
|
/// Whether `path` is `scope` or below it.
|
|
fn inside(path: &str, scope: &str) -> bool {
|
|
scope.is_empty()
|
|
|| path == scope
|
|
|| path
|
|
.strip_prefix(scope)
|
|
.is_some_and(|rest| rest.starts_with('/'))
|
|
}
|
|
|
|
/// Whether the directory `dir` is on the way to `scope` or inside it.
|
|
fn related(dir: &str, scope: &str) -> bool {
|
|
inside(dir, scope) || inside(scope, dir)
|
|
}
|
|
|
|
/// `path` relative to `root` with `/` separators; `None` outside the root
|
|
/// or for a name that is not valid Unicode.
|
|
fn relative(root: &Path, path: &Path) -> Option<String> {
|
|
let rel = path.strip_prefix(root).ok()?;
|
|
let parts: Option<Vec<&str>> = rel.components().map(|c| c.as_os_str().to_str()).collect();
|
|
Some(parts?.join("/"))
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
|
|
#[test]
|
|
fn scope_relations() {
|
|
assert!(inside("src/a.php", "src"));
|
|
assert!(inside("src", "src"));
|
|
assert!(!inside("src2/a.php", "src"));
|
|
assert!(inside("anything", ""));
|
|
assert!(related("src", "src/deep/a.php"));
|
|
assert!(related("src/deep/x", "src/deep"));
|
|
assert!(!related("lib", "src/a.php"));
|
|
}
|
|
}
|