M2.3: find in files backend
crates/search: parallel ignore walk + grep-searcher with the project's
ignore rules (vendor searched as library, node_modules and .git skipped),
literal or regex, case, whole word, include/exclude globs, binary and
> 8 MiB skips, match cap, cancel token, buffer overlay.
Engine: search.find returns {searchId} at once and runs off the lock;
pump_vfs drains results into searchResults (one per file, at most 500
per pump) and searchDone; search.cancel. Testkit: generated search tree.
Tests: library unit and property tests, engine tests, TCP control test.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
a6b2fd1300
commit
172693c8dd
21 changed files with 2231 additions and 5 deletions
345
crates/search/tests/search.rs
Normal file
345
crates/search/tests/search.rs
Normal file
|
|
@ -0,0 +1,345 @@
|
|||
//! Find in files over real directories: matching, rules, caps, cancel.
|
||||
|
||||
#![allow(
|
||||
clippy::unwrap_used,
|
||||
clippy::expect_used,
|
||||
clippy::indexing_slicing,
|
||||
clippy::panic
|
||||
)]
|
||||
|
||||
use std::fs;
|
||||
use std::path::Path;
|
||||
use std::sync::Mutex;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use lutra_search::{
|
||||
CancelToken, FileMatches, MAX_FILE_SIZE, Overlay, Query, Search, SearchError, SearchEvent,
|
||||
Summary, run, spawn,
|
||||
};
|
||||
use proptest::prelude::*;
|
||||
|
||||
/// Runs `query` over `root` and returns the files sorted by path.
|
||||
fn find(root: &Path, query: &Query, overlay: &Overlay) -> (Vec<FileMatches>, Summary) {
|
||||
let search = Search::new(query).unwrap();
|
||||
let found = Mutex::new(Vec::new());
|
||||
let summary = run(root, &search, overlay, &CancelToken::new(), &|f| {
|
||||
found.lock().unwrap().push(f)
|
||||
});
|
||||
let mut files = found.into_inner().unwrap();
|
||||
files.sort_by(|a, b| a.path.cmp(&b.path));
|
||||
(files, summary)
|
||||
}
|
||||
|
||||
fn paths(files: &[FileMatches]) -> Vec<&str> {
|
||||
files.iter().map(|f| f.path.as_str()).collect()
|
||||
}
|
||||
|
||||
fn put(root: &Path, rel: &str, text: &[u8]) {
|
||||
let path = root.join(rel);
|
||||
fs::create_dir_all(path.parent().unwrap()).unwrap();
|
||||
fs::write(path, text).unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn literal_case_and_whole_word() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
put(
|
||||
dir.path(),
|
||||
"a.php",
|
||||
b"<?php\n$user = new User();\n$users = [];\nUSER\n",
|
||||
);
|
||||
let mut q = Query::literal("user");
|
||||
let (files, summary) = find(dir.path(), &q, &Overlay::new());
|
||||
assert_eq!(summary.matches, 4);
|
||||
let m = &files[0].matches;
|
||||
assert_eq!((m[0].line, m[0].column, m[0].length), (1, 1, 4));
|
||||
assert_eq!(m[0].preview, "$user = new User();");
|
||||
assert_eq!((m[1].line, m[1].column), (1, 12));
|
||||
|
||||
q.case_sensitive = true;
|
||||
assert_eq!(find(dir.path(), &q, &Overlay::new()).1.matches, 2);
|
||||
|
||||
q.whole_word = true;
|
||||
let (files, _) = find(dir.path(), &q, &Overlay::new());
|
||||
assert_eq!(files[0].matches.len(), 1);
|
||||
assert_eq!(files[0].matches[0].line, 1);
|
||||
|
||||
let q = Query::literal("$users = [");
|
||||
let (files, _) = find(dir.path(), &q, &Overlay::new());
|
||||
assert_eq!(files[0].matches[0].line, 2, "regex characters are literal");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn regex_and_invalid_input() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
put(dir.path(), "a.php", b"foo1 foo22 bar\r\nfoo333\r\n");
|
||||
let mut q = Query::literal(r"foo\d+");
|
||||
q.regex = true;
|
||||
let (files, summary) = find(dir.path(), &q, &Overlay::new());
|
||||
assert_eq!(summary.matches, 3);
|
||||
assert_eq!(files[0].matches[2].preview, "foo333", "no \\r in previews");
|
||||
assert_eq!(files[0].matches[1].length, 5);
|
||||
|
||||
q.pattern = "(".into();
|
||||
assert!(matches!(
|
||||
Search::new(&q),
|
||||
Err(SearchError::InvalidPattern(_))
|
||||
));
|
||||
q.pattern = String::new();
|
||||
assert_eq!(Search::new(&q).unwrap_err(), SearchError::EmptyPattern);
|
||||
let mut q = Query::literal("x");
|
||||
q.include = vec!["[".into()];
|
||||
assert!(matches!(Search::new(&q), Err(SearchError::InvalidGlob(_))));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn include_and_exclude_globs() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
for rel in ["src/a.php", "src/b.js", "tests/c.php", "src/deep/d.php"] {
|
||||
put(dir.path(), rel, b"needle\n");
|
||||
}
|
||||
let mut q = Query::literal("needle");
|
||||
q.include = vec!["*.php".into()];
|
||||
let (files, _) = find(dir.path(), &q, &Overlay::new());
|
||||
assert_eq!(
|
||||
paths(&files),
|
||||
["src/a.php", "src/deep/d.php", "tests/c.php"]
|
||||
);
|
||||
|
||||
q.exclude = vec!["tests/".into()];
|
||||
let (files, _) = find(dir.path(), &q, &Overlay::new());
|
||||
assert_eq!(paths(&files), ["src/a.php", "src/deep/d.php"]);
|
||||
|
||||
q.include = vec!["src/*.php".into()];
|
||||
q.exclude = vec![];
|
||||
let (files, _) = find(dir.path(), &q, &Overlay::new());
|
||||
assert_eq!(
|
||||
paths(&files),
|
||||
["src/a.php"],
|
||||
"a path glob keeps to its level"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn honours_the_project_rules() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let tree = lutra_testkit::generate_search_tree(dir.path(), 400);
|
||||
put(dir.path(), "sub/.gitignore", b"skip.php\n");
|
||||
put(dir.path(), "sub/skip.php", b"lutraNeedle\n");
|
||||
put(dir.path(), "sub/.hidden.php", b"lutraNeedle\n");
|
||||
put(dir.path(), ".git/HEAD", b"lutraNeedle\n");
|
||||
let (files, summary) = find(
|
||||
dir.path(),
|
||||
&Query::literal(lutra_testkit::NEEDLE),
|
||||
&Overlay::new(),
|
||||
);
|
||||
let found = paths(&files);
|
||||
assert!(found.iter().all(|p| !p.starts_with("var/")), "{found:?}");
|
||||
assert!(found.iter().all(|p| !p.starts_with("node_modules/")));
|
||||
assert!(found.iter().all(|p| !p.starts_with(".git/")));
|
||||
assert!(!found.contains(&"bin/data.bin"), "binary skipped");
|
||||
assert!(!found.contains(&"debug.log"), "ignored file skipped");
|
||||
assert!(!found.contains(&"sub/skip.php"), "nested .gitignore");
|
||||
assert!(found.contains(&"sub/.hidden.php"), "dot files are searched");
|
||||
let vendor = found.iter().filter(|p| p.starts_with("vendor/")).count();
|
||||
assert_eq!(vendor, tree.needle_vendor, "vendor is searched");
|
||||
assert!(vendor > 0);
|
||||
assert_eq!(summary.files, tree.needle_files + 1);
|
||||
assert!(!summary.truncated && !summary.cancelled);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn skips_large_and_binary_files() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let mut big = b"needle\n".to_vec();
|
||||
big.resize(usize::try_from(MAX_FILE_SIZE).unwrap() + 1, b'a');
|
||||
put(dir.path(), "big.txt", &big);
|
||||
let mut late_nul = b"needle\n".to_vec();
|
||||
late_nul.extend(b"filler\n".repeat(50_000));
|
||||
late_nul.push(0);
|
||||
put(dir.path(), "late.bin", &late_nul);
|
||||
put(dir.path(), "ok.txt", b"needle\n");
|
||||
let (files, _) = find(dir.path(), &Query::literal("needle"), &Overlay::new());
|
||||
assert_eq!(paths(&files), ["ok.txt"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn overlay_replaces_the_disk_text() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
put(dir.path(), "a.php", b"old text\n");
|
||||
put(dir.path(), "b.php", b"needle on disk\n");
|
||||
let mut overlay = Overlay::new();
|
||||
overlay.insert("a.php".into(), "line\n needle in buffer\n".into());
|
||||
overlay.insert("b.php".into(), "gone\n".into());
|
||||
overlay.insert("missing.php".into(), "needle\n".into());
|
||||
let (files, _) = find(dir.path(), &Query::literal("needle"), &overlay);
|
||||
assert_eq!(paths(&files), ["a.php"]);
|
||||
assert_eq!(files[0].matches[0].line, 1);
|
||||
assert_eq!(files[0].matches[0].column, 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn caps_the_number_of_matches() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
for i in 0..20 {
|
||||
put(dir.path(), &format!("f{i}.txt"), b"x x x x x\n");
|
||||
}
|
||||
let mut q = Query::literal("x");
|
||||
q.max_results = 7;
|
||||
let (files, summary) = find(dir.path(), &q, &Overlay::new());
|
||||
assert_eq!(summary.matches, 7);
|
||||
assert!(summary.truncated);
|
||||
assert_eq!(files.iter().map(|f| f.matches.len()).sum::<usize>(), 7);
|
||||
assert_eq!(summary.files, files.len());
|
||||
|
||||
q.max_results = 100;
|
||||
let (_, summary) = find(dir.path(), &q, &Overlay::new());
|
||||
assert_eq!(summary.matches, 100);
|
||||
assert!(
|
||||
!summary.truncated,
|
||||
"exactly at the cap with nothing left out"
|
||||
);
|
||||
q.max_results = 99;
|
||||
assert!(find(dir.path(), &q, &Overlay::new()).1.truncated);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn spawn_streams_and_cancels() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let tree = lutra_testkit::generate_search_tree(dir.path(), 300);
|
||||
let search = Search::new(&Query::literal(lutra_testkit::NEEDLE)).unwrap();
|
||||
let rx = spawn(
|
||||
dir.path().into(),
|
||||
search.clone(),
|
||||
Overlay::new(),
|
||||
CancelToken::new(),
|
||||
)
|
||||
.unwrap();
|
||||
let events: Vec<SearchEvent> = rx.iter().collect();
|
||||
let files = events
|
||||
.iter()
|
||||
.filter(|e| matches!(e, SearchEvent::File(_)))
|
||||
.count();
|
||||
assert_eq!(files, tree.needle_files);
|
||||
let Some(SearchEvent::Done(summary)) = events.last() else {
|
||||
panic!("no summary last: {events:?}");
|
||||
};
|
||||
assert_eq!(summary.files, files);
|
||||
|
||||
let cancel = CancelToken::new();
|
||||
cancel.cancel();
|
||||
let rx = spawn(dir.path().into(), search, Overlay::new(), cancel).unwrap();
|
||||
let events: Vec<SearchEvent> = rx.iter().collect();
|
||||
assert_eq!(events.len(), 1);
|
||||
let SearchEvent::Done(summary) = &events[0] else {
|
||||
panic!("{events:?}");
|
||||
};
|
||||
assert!(summary.cancelled);
|
||||
assert_eq!(summary.matches, 0);
|
||||
}
|
||||
|
||||
/// Measures the first result and the total on a generated tree of 20,000
|
||||
/// files. Run in release: `cargo test -p lutra-search --release --test
|
||||
/// search -- --ignored --nocapture measure`.
|
||||
#[test]
|
||||
#[ignore = "measurement; run explicitly in release"]
|
||||
fn measure_20k_tree() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let made = Instant::now();
|
||||
let tree = lutra_testkit::generate_search_tree(dir.path(), 20_000);
|
||||
println!("generated {} files in {:?}", tree.files, made.elapsed());
|
||||
for (label, pattern) in [
|
||||
("common literal", "function"),
|
||||
("rare literal", lutra_testkit::NEEDLE),
|
||||
] {
|
||||
for round in 0..3 {
|
||||
let search = Search::new(&Query::literal(pattern)).unwrap();
|
||||
let started = Instant::now();
|
||||
let rx = spawn(
|
||||
dir.path().into(),
|
||||
search,
|
||||
Overlay::new(),
|
||||
CancelToken::new(),
|
||||
)
|
||||
.unwrap();
|
||||
let mut first: Option<Duration> = None;
|
||||
let mut summary = None;
|
||||
for event in rx {
|
||||
match event {
|
||||
SearchEvent::File(_) => {
|
||||
first.get_or_insert_with(|| started.elapsed());
|
||||
}
|
||||
SearchEvent::Done(s) => summary = Some(s),
|
||||
}
|
||||
}
|
||||
let s = summary.unwrap();
|
||||
println!(
|
||||
"{label} round {round}: first {:?}, total {:?}, files {}, matches {}, truncated {}",
|
||||
first.unwrap(),
|
||||
s.elapsed,
|
||||
s.files,
|
||||
s.matches,
|
||||
s.truncated
|
||||
);
|
||||
}
|
||||
}
|
||||
let mut q = Query::literal("function");
|
||||
q.max_results = lutra_search::MAX_RESULTS_LIMIT;
|
||||
let search = Search::new(&q).unwrap();
|
||||
let s = run(
|
||||
dir.path(),
|
||||
&search,
|
||||
&Overlay::new(),
|
||||
&CancelToken::new(),
|
||||
&|_| {},
|
||||
);
|
||||
println!(
|
||||
"common literal, cap 100k: total {:?}, files {}, matches {}, truncated {}",
|
||||
s.elapsed, s.files, s.matches, s.truncated
|
||||
);
|
||||
}
|
||||
|
||||
proptest! {
|
||||
#![proptest_config(ProptestConfig::with_cases(64))]
|
||||
|
||||
/// Random patterns, flags and file contents never panic, and every
|
||||
/// reported match lies inside its preview.
|
||||
#[test]
|
||||
fn random_queries_never_panic(
|
||||
pattern in "\\PC{0,12}",
|
||||
regex in any::<bool>(),
|
||||
case_sensitive in any::<bool>(),
|
||||
whole_word in any::<bool>(),
|
||||
content in proptest::collection::vec(any::<u8>(), 0..400),
|
||||
text in "\\PC{0,200}",
|
||||
) {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
fs::write(dir.path().join("bytes.txt"), &content).unwrap();
|
||||
fs::write(dir.path().join("text.php"), &text).unwrap();
|
||||
let query = Query {
|
||||
pattern,
|
||||
regex,
|
||||
case_sensitive,
|
||||
whole_word,
|
||||
include: vec![],
|
||||
exclude: vec![],
|
||||
max_results: 50,
|
||||
};
|
||||
let Ok(search) = Search::new(&query) else {
|
||||
return Ok(());
|
||||
};
|
||||
let found = Mutex::new(Vec::new());
|
||||
let summary = run(dir.path(), &search, &Overlay::new(), &CancelToken::new(), &|f| {
|
||||
found.lock().unwrap().push(f)
|
||||
});
|
||||
let found = found.into_inner().unwrap();
|
||||
prop_assert!(summary.matches <= 50);
|
||||
for file in found {
|
||||
for m in file.matches {
|
||||
prop_assert!(m.length > 0);
|
||||
let units = m.preview.encode_utf16().count();
|
||||
prop_assert!(m.preview_column + m.length <= units || m.preview_column <= units);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue