M2.3: find in files backend
crates/search: parallel ignore walk + grep-searcher with the project's
ignore rules (vendor searched as library, node_modules and .git skipped),
literal or regex, case, whole word, include/exclude globs, binary and
> 8 MiB skips, match cap, cancel token, buffer overlay.
Engine: search.find returns {searchId} at once and runs off the lock;
pump_vfs drains results into searchResults (one per file, at most 500
per pump) and searchDone; search.cancel. Testkit: generated search tree.
Tests: library unit and property tests, engine tests, TCP control test.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
a6b2fd1300
commit
172693c8dd
21 changed files with 2231 additions and 5 deletions
280
crates/search/src/walk.rs
Normal file
280
crates/search/src/walk.rs
Normal file
|
|
@ -0,0 +1,280 @@
|
|||
//! The parallel walk over the project and the per-file search.
|
||||
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
|
||||
use std::time::Instant;
|
||||
|
||||
use grep_matcher::LineTerminator;
|
||||
use grep_regex::RegexMatcher;
|
||||
use grep_searcher::{BinaryDetection, Searcher, SearcherBuilder, Sink, SinkMatch};
|
||||
use ignore::{DirEntry, WalkBuilder, WalkState};
|
||||
|
||||
use crate::query::PathFilter;
|
||||
use crate::{
|
||||
CancelToken, FileMatches, MAX_FILE_SIZE, MAX_THREADS, Match, Overlay, Search, Summary, lines,
|
||||
};
|
||||
|
||||
/// Searches the project at `root` and calls `on_file` for every file with
|
||||
/// matches, from the walker threads, while the search runs. Blocks until
|
||||
/// the walk ends, the cap is reached or `cancel` fires.
|
||||
///
|
||||
/// # Parameters
|
||||
/// - `root`: the project root (absolute).
|
||||
/// - `search`: the compiled query.
|
||||
/// - `overlay`: buffer texts that replace files on disk.
|
||||
/// - `cancel`: stops the search; checked between files and lines.
|
||||
/// - `on_file`: receives each file's matches.
|
||||
///
|
||||
/// # Returns
|
||||
/// The totals. Files that cannot be read are skipped silently: a search
|
||||
/// reports what it can read, like the project tree lists what it can.
|
||||
pub fn run(
|
||||
root: &Path,
|
||||
search: &Search,
|
||||
overlay: &Overlay,
|
||||
cancel: &CancelToken,
|
||||
on_file: &(dyn Fn(FileMatches) + Sync),
|
||||
) -> Summary {
|
||||
let started = Instant::now();
|
||||
let totals = Totals::default();
|
||||
walker(root, &search.filter).build_parallel().run(|| {
|
||||
let mut searcher = SearcherBuilder::new()
|
||||
.line_terminator(LineTerminator::crlf())
|
||||
.line_number(true)
|
||||
.binary_detection(BinaryDetection::quit(0))
|
||||
.build();
|
||||
let totals = &totals;
|
||||
Box::new(move |entry| {
|
||||
if cancel.is_cancelled() || totals.full.load(Ordering::Relaxed) {
|
||||
return WalkState::Quit;
|
||||
}
|
||||
let Ok(entry) = entry else {
|
||||
return WalkState::Continue;
|
||||
};
|
||||
let Some(rel) = file_path(root, &entry, &search.filter) else {
|
||||
return WalkState::Continue;
|
||||
};
|
||||
let room = search
|
||||
.max_results
|
||||
.saturating_sub(totals.matches.load(Ordering::Relaxed));
|
||||
let Some(mut matches) = search_file(
|
||||
&mut searcher,
|
||||
&search.matcher,
|
||||
entry.path(),
|
||||
overlay.get(&rel),
|
||||
room,
|
||||
cancel,
|
||||
) else {
|
||||
return WalkState::Continue;
|
||||
};
|
||||
let granted = totals.reserve(matches.len(), search.max_results);
|
||||
if granted < matches.len() {
|
||||
matches.truncate(granted);
|
||||
totals.full.store(true, Ordering::Relaxed);
|
||||
}
|
||||
if granted > 0 {
|
||||
totals.files.fetch_add(1, Ordering::Relaxed);
|
||||
on_file(FileMatches { path: rel, matches });
|
||||
}
|
||||
if totals.full.load(Ordering::Relaxed) {
|
||||
WalkState::Quit
|
||||
} else {
|
||||
WalkState::Continue
|
||||
}
|
||||
})
|
||||
});
|
||||
let cancelled = cancel.is_cancelled();
|
||||
Summary {
|
||||
files: totals.files.load(Ordering::Relaxed),
|
||||
matches: totals.matches.load(Ordering::Relaxed),
|
||||
truncated: totals.full.load(Ordering::Relaxed) && !cancelled,
|
||||
cancelled,
|
||||
elapsed: started.elapsed(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Counters shared by the walker threads.
|
||||
#[derive(Default)]
|
||||
struct Totals {
|
||||
/// Files reported.
|
||||
files: AtomicUsize,
|
||||
/// Matches reported.
|
||||
matches: AtomicUsize,
|
||||
/// The cap was reached, or a file had more matches than fit.
|
||||
full: AtomicBool,
|
||||
}
|
||||
|
||||
impl Totals {
|
||||
/// Takes up to `wanted` matches from what is left under `cap` and
|
||||
/// returns how many were granted. The caller marks the totals full when
|
||||
/// it got fewer than it wanted: only then is a match known to be left
|
||||
/// out.
|
||||
fn reserve(&self, wanted: usize, cap: usize) -> usize {
|
||||
let mut used = self.matches.load(Ordering::Relaxed);
|
||||
loop {
|
||||
let granted = wanted.min(cap.saturating_sub(used));
|
||||
match self.matches.compare_exchange_weak(
|
||||
used,
|
||||
used.saturating_add(granted),
|
||||
Ordering::Relaxed,
|
||||
Ordering::Relaxed,
|
||||
) {
|
||||
Ok(_) => return granted,
|
||||
Err(now) => used = now,
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// The walker over `root` with the project's rules (see the crate docs).
|
||||
fn walker(root: &Path, filter: &PathFilter) -> WalkBuilder {
|
||||
let threads = std::thread::available_parallelism()
|
||||
.map(|n| n.get())
|
||||
.unwrap_or(2)
|
||||
.clamp(1, MAX_THREADS);
|
||||
let mut builder = WalkBuilder::new(root);
|
||||
builder
|
||||
.hidden(false)
|
||||
.parents(false)
|
||||
.ignore(true)
|
||||
.git_ignore(true)
|
||||
.git_exclude(true)
|
||||
.git_global(true)
|
||||
.require_git(false)
|
||||
.follow_links(false)
|
||||
.threads(threads);
|
||||
let vendor = root.join("vendor");
|
||||
let vendor_is_dir = std::fs::symlink_metadata(&vendor).is_ok_and(|m| m.is_dir());
|
||||
if vendor_is_dir {
|
||||
builder.add(&vendor);
|
||||
}
|
||||
let root = root.to_path_buf();
|
||||
let filter = filter.clone();
|
||||
builder.filter_entry(move |entry| keep_entry(&root, &vendor, &filter, entry));
|
||||
builder
|
||||
}
|
||||
|
||||
/// Whether the walk enters (directory) or visits (file) `entry`: `.git` and
|
||||
/// `node_modules` are skipped, the root's `vendor` is reached through its
|
||||
/// own walk root instead of the project root's walk, and exclude globs drop
|
||||
/// whole directories.
|
||||
fn keep_entry(root: &Path, vendor: &Path, filter: &PathFilter, entry: &DirEntry) -> bool {
|
||||
let name = entry.file_name();
|
||||
if name == ".git" {
|
||||
return false;
|
||||
}
|
||||
let is_dir = entry.file_type().is_some_and(|t| t.is_dir());
|
||||
if is_dir && name == "node_modules" {
|
||||
return false;
|
||||
}
|
||||
if entry.depth() == 1 && entry.path() == vendor {
|
||||
return false;
|
||||
}
|
||||
if is_dir && let Some(rel) = relative(root, entry.path()) {
|
||||
return !filter.excluded(&rel);
|
||||
}
|
||||
true
|
||||
}
|
||||
|
||||
/// The project-relative path of `entry` if it is a regular file that the
|
||||
/// globs keep.
|
||||
fn file_path(root: &Path, entry: &DirEntry, filter: &PathFilter) -> Option<String> {
|
||||
if !entry.file_type().is_some_and(|t| t.is_file()) {
|
||||
return None;
|
||||
}
|
||||
let rel = relative(root, entry.path())?;
|
||||
filter.file(&rel).then_some(rel)
|
||||
}
|
||||
|
||||
/// `path` relative to `root` with `/` separators; `None` outside the root or
|
||||
/// for a name that is not valid Unicode.
|
||||
fn relative(root: &Path, path: &Path) -> Option<String> {
|
||||
let rel: PathBuf = path.strip_prefix(root).ok()?.to_path_buf();
|
||||
let parts: Option<Vec<&str>> = rel.components().map(|c| c.as_os_str().to_str()).collect();
|
||||
Some(parts?.join("/"))
|
||||
}
|
||||
|
||||
/// Searches one file, or the buffer text that replaces it, and returns its
|
||||
/// matches; `None` without matches, for a binary, oversized or unreadable
|
||||
/// file, or when cancelled.
|
||||
///
|
||||
/// # Parameters
|
||||
/// - `room`: most matches to collect; one more is collected when possible,
|
||||
/// so the caller can tell that the cap cut the file short (with no room
|
||||
/// left, one match is enough to know that).
|
||||
fn search_file(
|
||||
searcher: &mut Searcher,
|
||||
matcher: &RegexMatcher,
|
||||
path: &Path,
|
||||
buffer: Option<&String>,
|
||||
room: usize,
|
||||
cancel: &CancelToken,
|
||||
) -> Option<Vec<Match>> {
|
||||
let mut sink = Collect {
|
||||
matcher,
|
||||
matches: Vec::new(),
|
||||
limit: room.saturating_add(1),
|
||||
binary: false,
|
||||
cancel,
|
||||
};
|
||||
let result = match buffer {
|
||||
Some(text) => {
|
||||
if text.len() as u64 > MAX_FILE_SIZE {
|
||||
return None;
|
||||
}
|
||||
searcher.search_slice(matcher, text.as_bytes(), &mut sink)
|
||||
}
|
||||
None => {
|
||||
let size = std::fs::metadata(path).ok()?.len();
|
||||
if size > MAX_FILE_SIZE {
|
||||
return None;
|
||||
}
|
||||
searcher.search_path(matcher, path, &mut sink)
|
||||
}
|
||||
};
|
||||
if result.is_err() || sink.binary || cancel.is_cancelled() || sink.matches.is_empty() {
|
||||
return None;
|
||||
}
|
||||
Some(sink.matches)
|
||||
}
|
||||
|
||||
/// Collects the matches of one file.
|
||||
struct Collect<'a> {
|
||||
/// The pattern, run again on each matching line to find every match.
|
||||
matcher: &'a RegexMatcher,
|
||||
/// Matches so far.
|
||||
matches: Vec<Match>,
|
||||
/// Stop at this many matches.
|
||||
limit: usize,
|
||||
/// The file turned out to be binary.
|
||||
binary: bool,
|
||||
/// Stops the file early.
|
||||
cancel: &'a CancelToken,
|
||||
}
|
||||
|
||||
impl Sink for Collect<'_> {
|
||||
type Error = std::io::Error;
|
||||
|
||||
fn matched(&mut self, _searcher: &Searcher, mat: &SinkMatch<'_>) -> Result<bool, Self::Error> {
|
||||
if self.cancel.is_cancelled() {
|
||||
return Ok(false);
|
||||
}
|
||||
let line = mat
|
||||
.line_number()
|
||||
.and_then(|n| usize::try_from(n.saturating_sub(1)).ok())
|
||||
.unwrap_or(0);
|
||||
lines::collect(
|
||||
self.matcher,
|
||||
line,
|
||||
mat.bytes(),
|
||||
&mut self.matches,
|
||||
self.limit,
|
||||
);
|
||||
Ok(self.matches.len() < self.limit)
|
||||
}
|
||||
|
||||
fn binary_data(&mut self, _searcher: &Searcher, _offset: u64) -> Result<bool, Self::Error> {
|
||||
self.binary = true;
|
||||
Ok(false)
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue