crates/search: parallel ignore walk + grep-searcher with the project's
ignore rules (vendor searched as library, node_modules and .git skipped),
literal or regex, case, whole word, include/exclude globs, binary and
> 8 MiB skips, match cap, cancel token, buffer overlay.
Engine: search.find returns {searchId} at once and runs off the lock;
pump_vfs drains results into searchResults (one per file, at most 500
per pump) and searchDone; search.cancel. Testkit: generated search tree.
Tests: library unit and property tests, engine tests, TCP control test.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
139 lines
4.6 KiB
Rust
139 lines
4.6 KiB
Rust
//! Compiling a [`Query`]: the matcher and the path filters.
|
|
|
|
use globset::{GlobBuilder, GlobSet, GlobSetBuilder};
|
|
use grep_regex::{RegexMatcher, RegexMatcherBuilder};
|
|
|
|
use crate::{MAX_RESULTS_LIMIT, Query, SearchError};
|
|
|
|
/// Largest compiled regex, in bytes; bigger patterns are rejected instead of
|
|
/// using unbounded memory.
|
|
const REGEX_SIZE_LIMIT: usize = 16 * 1024 * 1024;
|
|
|
|
/// Most nesting a pattern may use.
|
|
const REGEX_NEST_LIMIT: u32 = 100;
|
|
|
|
/// A compiled query, ready to run.
|
|
#[derive(Debug, Clone)]
|
|
pub struct Search {
|
|
/// Matches one line at a time.
|
|
pub(crate) matcher: RegexMatcher,
|
|
/// Include and exclude globs.
|
|
pub(crate) filter: PathFilter,
|
|
/// Cap on reported matches.
|
|
pub(crate) max_results: usize,
|
|
}
|
|
|
|
impl Search {
|
|
/// Compiles `query`.
|
|
///
|
|
/// # Errors
|
|
/// [`SearchError::EmptyPattern`], [`SearchError::InvalidPattern`] for a
|
|
/// bad regular expression (or one that can match a line break),
|
|
/// [`SearchError::InvalidGlob`] for a bad include or exclude glob.
|
|
pub fn new(query: &Query) -> Result<Self, SearchError> {
|
|
if query.pattern.is_empty() {
|
|
return Err(SearchError::EmptyPattern);
|
|
}
|
|
let matcher = RegexMatcherBuilder::new()
|
|
.fixed_strings(!query.regex)
|
|
.case_insensitive(!query.case_sensitive)
|
|
.word(query.whole_word)
|
|
.line_terminator(Some(b'\n'))
|
|
.crlf(true)
|
|
.size_limit(REGEX_SIZE_LIMIT)
|
|
.dfa_size_limit(REGEX_SIZE_LIMIT)
|
|
.nest_limit(REGEX_NEST_LIMIT)
|
|
.build(&query.pattern)
|
|
.map_err(|e| SearchError::InvalidPattern(e.to_string()))?;
|
|
Ok(Self {
|
|
matcher,
|
|
filter: PathFilter::new(&query.include, &query.exclude)?,
|
|
max_results: query.max_results.clamp(1, MAX_RESULTS_LIMIT),
|
|
})
|
|
}
|
|
}
|
|
|
|
/// Include and exclude globs, split by whether they match a name or a path.
|
|
#[derive(Debug, Clone)]
|
|
pub(crate) struct PathFilter {
|
|
/// Include globs; `None` when every file is included.
|
|
include: Option<Globs>,
|
|
/// Exclude globs; `None` when nothing is excluded.
|
|
exclude: Option<Globs>,
|
|
}
|
|
|
|
/// Globs matched against the file name and against the relative path.
|
|
#[derive(Debug, Clone)]
|
|
struct Globs {
|
|
/// Globs without `/`.
|
|
names: GlobSet,
|
|
/// Globs with `/`.
|
|
paths: GlobSet,
|
|
}
|
|
|
|
impl PathFilter {
|
|
/// Compiles the globs; blank ones are ignored.
|
|
fn new(include: &[String], exclude: &[String]) -> Result<Self, SearchError> {
|
|
Ok(Self {
|
|
include: Globs::new(include)?,
|
|
exclude: Globs::new(exclude)?,
|
|
})
|
|
}
|
|
|
|
/// Whether the file at `rel` (project-relative, `/` separators) passes
|
|
/// the include and exclude globs.
|
|
pub(crate) fn file(&self, rel: &str) -> bool {
|
|
let included = self.include.as_ref().is_none_or(|g| g.matches(rel));
|
|
included && !self.excluded(rel)
|
|
}
|
|
|
|
/// Whether an exclude glob matches the file or directory at `rel`.
|
|
pub(crate) fn excluded(&self, rel: &str) -> bool {
|
|
self.exclude.as_ref().is_some_and(|g| g.matches(rel))
|
|
}
|
|
}
|
|
|
|
impl Globs {
|
|
/// Compiles `globs`; `None` if there is none. Leading and trailing `/`
|
|
/// are dropped, so `tests/` excludes every directory named `tests`.
|
|
fn new(globs: &[String]) -> Result<Option<Self>, SearchError> {
|
|
let mut names = GlobSetBuilder::new();
|
|
let mut paths = GlobSetBuilder::new();
|
|
let mut any = false;
|
|
for glob in globs.iter().map(|g| g.trim()).filter(|g| !g.is_empty()) {
|
|
let pattern = glob.trim_matches('/');
|
|
if pattern.is_empty() {
|
|
continue;
|
|
}
|
|
let compiled = GlobBuilder::new(pattern)
|
|
.literal_separator(true)
|
|
.backslash_escape(true)
|
|
.build()
|
|
.map_err(|e| SearchError::InvalidGlob(e.to_string()))?;
|
|
if pattern.contains('/') {
|
|
paths.add(compiled);
|
|
} else {
|
|
names.add(compiled);
|
|
}
|
|
any = true;
|
|
}
|
|
if !any {
|
|
return Ok(None);
|
|
}
|
|
let build = |b: GlobSetBuilder| {
|
|
b.build()
|
|
.map_err(|e| SearchError::InvalidGlob(e.to_string()))
|
|
};
|
|
Ok(Some(Self {
|
|
names: build(names)?,
|
|
paths: build(paths)?,
|
|
}))
|
|
}
|
|
|
|
/// Whether `rel`'s last component matches a name glob or `rel` a path
|
|
/// glob.
|
|
fn matches(&self, rel: &str) -> bool {
|
|
let name = rel.rsplit('/').next().unwrap_or(rel);
|
|
self.names.is_match(name) || self.paths.is_match(rel)
|
|
}
|
|
}
|