//! The parallel walk over the project and the per-file search. use std::path::{Path, PathBuf}; use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; use std::time::Instant; use grep_matcher::LineTerminator; use grep_regex::RegexMatcher; use grep_searcher::{BinaryDetection, Searcher, SearcherBuilder, Sink, SinkMatch}; use ignore::{DirEntry, WalkBuilder, WalkState}; use crate::query::PathFilter; use crate::{ CancelToken, FileMatches, MAX_FILE_SIZE, MAX_THREADS, Match, Overlay, Search, Summary, lines, }; /// Searches the project at `root` and calls `on_file` for every file with /// matches, from the walker threads, while the search runs. Blocks until /// the walk ends, the cap is reached or `cancel` fires. /// /// # Parameters /// - `root`: the project root (absolute). /// - `search`: the compiled query. /// - `overlay`: buffer texts that replace files on disk. /// - `cancel`: stops the search; checked between files and lines. /// - `on_file`: receives each file's matches. /// /// # Returns /// The totals. Files that cannot be read are skipped silently: a search /// reports what it can read, like the project tree lists what it can. pub fn run( root: &Path, search: &Search, overlay: &Overlay, cancel: &CancelToken, on_file: &(dyn Fn(FileMatches) + Sync), ) -> Summary { let started = Instant::now(); let totals = Totals::default(); walker(root, &search.filter).build_parallel().run(|| { let mut searcher = SearcherBuilder::new() .line_terminator(LineTerminator::crlf()) .line_number(true) .binary_detection(BinaryDetection::quit(0)) .build(); let totals = &totals; Box::new(move |entry| { if cancel.is_cancelled() || totals.full.load(Ordering::Relaxed) { return WalkState::Quit; } let Ok(entry) = entry else { return WalkState::Continue; }; let Some(rel) = file_path(root, &entry, &search.filter) else { return WalkState::Continue; }; let room = search .max_results .saturating_sub(totals.matches.load(Ordering::Relaxed)); let Some(mut matches) = search_file( &mut searcher, &search.matcher, entry.path(), overlay.get(&rel), room, cancel, ) else { return WalkState::Continue; }; let granted = totals.reserve(matches.len(), search.max_results); if granted < matches.len() { matches.truncate(granted); totals.full.store(true, Ordering::Relaxed); } if granted > 0 { totals.files.fetch_add(1, Ordering::Relaxed); on_file(FileMatches { path: rel, matches }); } if totals.full.load(Ordering::Relaxed) { WalkState::Quit } else { WalkState::Continue } }) }); let cancelled = cancel.is_cancelled(); Summary { files: totals.files.load(Ordering::Relaxed), matches: totals.matches.load(Ordering::Relaxed), truncated: totals.full.load(Ordering::Relaxed) && !cancelled, cancelled, elapsed: started.elapsed(), } } /// Counters shared by the walker threads. #[derive(Default)] struct Totals { /// Files reported. files: AtomicUsize, /// Matches reported. matches: AtomicUsize, /// The cap was reached, or a file had more matches than fit. full: AtomicBool, } impl Totals { /// Takes up to `wanted` matches from what is left under `cap` and /// returns how many were granted. The caller marks the totals full when /// it got fewer than it wanted: only then is a match known to be left /// out. fn reserve(&self, wanted: usize, cap: usize) -> usize { let mut used = self.matches.load(Ordering::Relaxed); loop { let granted = wanted.min(cap.saturating_sub(used)); match self.matches.compare_exchange_weak( used, used.saturating_add(granted), Ordering::Relaxed, Ordering::Relaxed, ) { Ok(_) => return granted, Err(now) => used = now, } } } } /// The walker over `root` with the project's rules (see the crate docs). fn walker(root: &Path, filter: &PathFilter) -> WalkBuilder { let threads = std::thread::available_parallelism() .map(|n| n.get()) .unwrap_or(2) .clamp(1, MAX_THREADS); let mut builder = WalkBuilder::new(root); builder .hidden(false) .parents(false) .ignore(true) .git_ignore(true) .git_exclude(true) .git_global(true) .require_git(false) .follow_links(false) .threads(threads); let vendor = root.join("vendor"); let vendor_is_dir = std::fs::symlink_metadata(&vendor).is_ok_and(|m| m.is_dir()); if vendor_is_dir { builder.add(&vendor); } let root = root.to_path_buf(); let filter = filter.clone(); builder.filter_entry(move |entry| keep_entry(&root, &vendor, &filter, entry)); builder } /// Whether the walk enters (directory) or visits (file) `entry`: `.git` and /// `node_modules` are skipped, the root's `vendor` is reached through its /// own walk root instead of the project root's walk, and exclude globs drop /// whole directories. fn keep_entry(root: &Path, vendor: &Path, filter: &PathFilter, entry: &DirEntry) -> bool { let name = entry.file_name(); if name == ".git" { return false; } let is_dir = entry.file_type().is_some_and(|t| t.is_dir()); if is_dir && name == "node_modules" { return false; } if entry.depth() == 1 && entry.path() == vendor { return false; } if is_dir && let Some(rel) = relative(root, entry.path()) { return !filter.excluded(&rel); } true } /// The project-relative path of `entry` if it is a regular file that the /// globs keep. fn file_path(root: &Path, entry: &DirEntry, filter: &PathFilter) -> Option { if !entry.file_type().is_some_and(|t| t.is_file()) { return None; } let rel = relative(root, entry.path())?; filter.file(&rel).then_some(rel) } /// `path` relative to `root` with `/` separators; `None` outside the root or /// for a name that is not valid Unicode. fn relative(root: &Path, path: &Path) -> Option { let rel: PathBuf = path.strip_prefix(root).ok()?.to_path_buf(); let parts: Option> = rel.components().map(|c| c.as_os_str().to_str()).collect(); Some(parts?.join("/")) } /// Searches one file, or the buffer text that replaces it, and returns its /// matches; `None` without matches, for a binary, oversized or unreadable /// file, or when cancelled. /// /// # Parameters /// - `room`: most matches to collect; one more is collected when possible, /// so the caller can tell that the cap cut the file short (with no room /// left, one match is enough to know that). fn search_file( searcher: &mut Searcher, matcher: &RegexMatcher, path: &Path, buffer: Option<&String>, room: usize, cancel: &CancelToken, ) -> Option> { let mut sink = Collect { matcher, matches: Vec::new(), limit: room.saturating_add(1), binary: false, cancel, }; let result = match buffer { Some(text) => { if text.len() as u64 > MAX_FILE_SIZE { return None; } searcher.search_slice(matcher, text.as_bytes(), &mut sink) } None => { let size = std::fs::metadata(path).ok()?.len(); if size > MAX_FILE_SIZE { return None; } searcher.search_path(matcher, path, &mut sink) } }; if result.is_err() || sink.binary || cancel.is_cancelled() || sink.matches.is_empty() { return None; } Some(sink.matches) } /// Collects the matches of one file. struct Collect<'a> { /// The pattern, run again on each matching line to find every match. matcher: &'a RegexMatcher, /// Matches so far. matches: Vec, /// Stop at this many matches. limit: usize, /// The file turned out to be binary. binary: bool, /// Stops the file early. cancel: &'a CancelToken, } impl Sink for Collect<'_> { type Error = std::io::Error; fn matched(&mut self, _searcher: &Searcher, mat: &SinkMatch<'_>) -> Result { if self.cancel.is_cancelled() { return Ok(false); } let line = mat .line_number() .and_then(|n| usize::try_from(n.saturating_sub(1)).ok()) .unwrap_or(0); lines::collect( self.matcher, line, mat.bytes(), &mut self.matches, self.limit, ); Ok(self.matches.len() < self.limit) } fn binary_data(&mut self, _searcher: &Searcher, _offset: u64) -> Result { self.binary = true; Ok(false) } }