Lutra/crates/search/tests/search.rs
danny 172693c8dd M2.3: find in files backend
crates/search: parallel ignore walk + grep-searcher with the project's
ignore rules (vendor searched as library, node_modules and .git skipped),
literal or regex, case, whole word, include/exclude globs, binary and
> 8 MiB skips, match cap, cancel token, buffer overlay.

Engine: search.find returns {searchId} at once and runs off the lock;
pump_vfs drains results into searchResults (one per file, at most 500
per pump) and searchDone; search.cancel. Testkit: generated search tree.
Tests: library unit and property tests, engine tests, TCP control test.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-16 22:04:28 +02:00

345 lines
12 KiB
Rust

//! Find in files over real directories: matching, rules, caps, cancel.
#![allow(
clippy::unwrap_used,
clippy::expect_used,
clippy::indexing_slicing,
clippy::panic
)]
use std::fs;
use std::path::Path;
use std::sync::Mutex;
use std::time::{Duration, Instant};
use lutra_search::{
CancelToken, FileMatches, MAX_FILE_SIZE, Overlay, Query, Search, SearchError, SearchEvent,
Summary, run, spawn,
};
use proptest::prelude::*;
/// Runs `query` over `root` and returns the files sorted by path.
fn find(root: &Path, query: &Query, overlay: &Overlay) -> (Vec<FileMatches>, Summary) {
let search = Search::new(query).unwrap();
let found = Mutex::new(Vec::new());
let summary = run(root, &search, overlay, &CancelToken::new(), &|f| {
found.lock().unwrap().push(f)
});
let mut files = found.into_inner().unwrap();
files.sort_by(|a, b| a.path.cmp(&b.path));
(files, summary)
}
fn paths(files: &[FileMatches]) -> Vec<&str> {
files.iter().map(|f| f.path.as_str()).collect()
}
fn put(root: &Path, rel: &str, text: &[u8]) {
let path = root.join(rel);
fs::create_dir_all(path.parent().unwrap()).unwrap();
fs::write(path, text).unwrap();
}
#[test]
fn literal_case_and_whole_word() {
let dir = tempfile::tempdir().unwrap();
put(
dir.path(),
"a.php",
b"<?php\n$user = new User();\n$users = [];\nUSER\n",
);
let mut q = Query::literal("user");
let (files, summary) = find(dir.path(), &q, &Overlay::new());
assert_eq!(summary.matches, 4);
let m = &files[0].matches;
assert_eq!((m[0].line, m[0].column, m[0].length), (1, 1, 4));
assert_eq!(m[0].preview, "$user = new User();");
assert_eq!((m[1].line, m[1].column), (1, 12));
q.case_sensitive = true;
assert_eq!(find(dir.path(), &q, &Overlay::new()).1.matches, 2);
q.whole_word = true;
let (files, _) = find(dir.path(), &q, &Overlay::new());
assert_eq!(files[0].matches.len(), 1);
assert_eq!(files[0].matches[0].line, 1);
let q = Query::literal("$users = [");
let (files, _) = find(dir.path(), &q, &Overlay::new());
assert_eq!(files[0].matches[0].line, 2, "regex characters are literal");
}
#[test]
fn regex_and_invalid_input() {
let dir = tempfile::tempdir().unwrap();
put(dir.path(), "a.php", b"foo1 foo22 bar\r\nfoo333\r\n");
let mut q = Query::literal(r"foo\d+");
q.regex = true;
let (files, summary) = find(dir.path(), &q, &Overlay::new());
assert_eq!(summary.matches, 3);
assert_eq!(files[0].matches[2].preview, "foo333", "no \\r in previews");
assert_eq!(files[0].matches[1].length, 5);
q.pattern = "(".into();
assert!(matches!(
Search::new(&q),
Err(SearchError::InvalidPattern(_))
));
q.pattern = String::new();
assert_eq!(Search::new(&q).unwrap_err(), SearchError::EmptyPattern);
let mut q = Query::literal("x");
q.include = vec!["[".into()];
assert!(matches!(Search::new(&q), Err(SearchError::InvalidGlob(_))));
}
#[test]
fn include_and_exclude_globs() {
let dir = tempfile::tempdir().unwrap();
for rel in ["src/a.php", "src/b.js", "tests/c.php", "src/deep/d.php"] {
put(dir.path(), rel, b"needle\n");
}
let mut q = Query::literal("needle");
q.include = vec!["*.php".into()];
let (files, _) = find(dir.path(), &q, &Overlay::new());
assert_eq!(
paths(&files),
["src/a.php", "src/deep/d.php", "tests/c.php"]
);
q.exclude = vec!["tests/".into()];
let (files, _) = find(dir.path(), &q, &Overlay::new());
assert_eq!(paths(&files), ["src/a.php", "src/deep/d.php"]);
q.include = vec!["src/*.php".into()];
q.exclude = vec![];
let (files, _) = find(dir.path(), &q, &Overlay::new());
assert_eq!(
paths(&files),
["src/a.php"],
"a path glob keeps to its level"
);
}
#[test]
fn honours_the_project_rules() {
let dir = tempfile::tempdir().unwrap();
let tree = lutra_testkit::generate_search_tree(dir.path(), 400);
put(dir.path(), "sub/.gitignore", b"skip.php\n");
put(dir.path(), "sub/skip.php", b"lutraNeedle\n");
put(dir.path(), "sub/.hidden.php", b"lutraNeedle\n");
put(dir.path(), ".git/HEAD", b"lutraNeedle\n");
let (files, summary) = find(
dir.path(),
&Query::literal(lutra_testkit::NEEDLE),
&Overlay::new(),
);
let found = paths(&files);
assert!(found.iter().all(|p| !p.starts_with("var/")), "{found:?}");
assert!(found.iter().all(|p| !p.starts_with("node_modules/")));
assert!(found.iter().all(|p| !p.starts_with(".git/")));
assert!(!found.contains(&"bin/data.bin"), "binary skipped");
assert!(!found.contains(&"debug.log"), "ignored file skipped");
assert!(!found.contains(&"sub/skip.php"), "nested .gitignore");
assert!(found.contains(&"sub/.hidden.php"), "dot files are searched");
let vendor = found.iter().filter(|p| p.starts_with("vendor/")).count();
assert_eq!(vendor, tree.needle_vendor, "vendor is searched");
assert!(vendor > 0);
assert_eq!(summary.files, tree.needle_files + 1);
assert!(!summary.truncated && !summary.cancelled);
}
#[test]
fn skips_large_and_binary_files() {
let dir = tempfile::tempdir().unwrap();
let mut big = b"needle\n".to_vec();
big.resize(usize::try_from(MAX_FILE_SIZE).unwrap() + 1, b'a');
put(dir.path(), "big.txt", &big);
let mut late_nul = b"needle\n".to_vec();
late_nul.extend(b"filler\n".repeat(50_000));
late_nul.push(0);
put(dir.path(), "late.bin", &late_nul);
put(dir.path(), "ok.txt", b"needle\n");
let (files, _) = find(dir.path(), &Query::literal("needle"), &Overlay::new());
assert_eq!(paths(&files), ["ok.txt"]);
}
#[test]
fn overlay_replaces_the_disk_text() {
let dir = tempfile::tempdir().unwrap();
put(dir.path(), "a.php", b"old text\n");
put(dir.path(), "b.php", b"needle on disk\n");
let mut overlay = Overlay::new();
overlay.insert("a.php".into(), "line\n needle in buffer\n".into());
overlay.insert("b.php".into(), "gone\n".into());
overlay.insert("missing.php".into(), "needle\n".into());
let (files, _) = find(dir.path(), &Query::literal("needle"), &overlay);
assert_eq!(paths(&files), ["a.php"]);
assert_eq!(files[0].matches[0].line, 1);
assert_eq!(files[0].matches[0].column, 2);
}
#[test]
fn caps_the_number_of_matches() {
let dir = tempfile::tempdir().unwrap();
for i in 0..20 {
put(dir.path(), &format!("f{i}.txt"), b"x x x x x\n");
}
let mut q = Query::literal("x");
q.max_results = 7;
let (files, summary) = find(dir.path(), &q, &Overlay::new());
assert_eq!(summary.matches, 7);
assert!(summary.truncated);
assert_eq!(files.iter().map(|f| f.matches.len()).sum::<usize>(), 7);
assert_eq!(summary.files, files.len());
q.max_results = 100;
let (_, summary) = find(dir.path(), &q, &Overlay::new());
assert_eq!(summary.matches, 100);
assert!(
!summary.truncated,
"exactly at the cap with nothing left out"
);
q.max_results = 99;
assert!(find(dir.path(), &q, &Overlay::new()).1.truncated);
}
#[test]
fn spawn_streams_and_cancels() {
let dir = tempfile::tempdir().unwrap();
let tree = lutra_testkit::generate_search_tree(dir.path(), 300);
let search = Search::new(&Query::literal(lutra_testkit::NEEDLE)).unwrap();
let rx = spawn(
dir.path().into(),
search.clone(),
Overlay::new(),
CancelToken::new(),
)
.unwrap();
let events: Vec<SearchEvent> = rx.iter().collect();
let files = events
.iter()
.filter(|e| matches!(e, SearchEvent::File(_)))
.count();
assert_eq!(files, tree.needle_files);
let Some(SearchEvent::Done(summary)) = events.last() else {
panic!("no summary last: {events:?}");
};
assert_eq!(summary.files, files);
let cancel = CancelToken::new();
cancel.cancel();
let rx = spawn(dir.path().into(), search, Overlay::new(), cancel).unwrap();
let events: Vec<SearchEvent> = rx.iter().collect();
assert_eq!(events.len(), 1);
let SearchEvent::Done(summary) = &events[0] else {
panic!("{events:?}");
};
assert!(summary.cancelled);
assert_eq!(summary.matches, 0);
}
/// Measures the first result and the total on a generated tree of 20,000
/// files. Run in release: `cargo test -p lutra-search --release --test
/// search -- --ignored --nocapture measure`.
#[test]
#[ignore = "measurement; run explicitly in release"]
fn measure_20k_tree() {
let dir = tempfile::tempdir().unwrap();
let made = Instant::now();
let tree = lutra_testkit::generate_search_tree(dir.path(), 20_000);
println!("generated {} files in {:?}", tree.files, made.elapsed());
for (label, pattern) in [
("common literal", "function"),
("rare literal", lutra_testkit::NEEDLE),
] {
for round in 0..3 {
let search = Search::new(&Query::literal(pattern)).unwrap();
let started = Instant::now();
let rx = spawn(
dir.path().into(),
search,
Overlay::new(),
CancelToken::new(),
)
.unwrap();
let mut first: Option<Duration> = None;
let mut summary = None;
for event in rx {
match event {
SearchEvent::File(_) => {
first.get_or_insert_with(|| started.elapsed());
}
SearchEvent::Done(s) => summary = Some(s),
}
}
let s = summary.unwrap();
println!(
"{label} round {round}: first {:?}, total {:?}, files {}, matches {}, truncated {}",
first.unwrap(),
s.elapsed,
s.files,
s.matches,
s.truncated
);
}
}
let mut q = Query::literal("function");
q.max_results = lutra_search::MAX_RESULTS_LIMIT;
let search = Search::new(&q).unwrap();
let s = run(
dir.path(),
&search,
&Overlay::new(),
&CancelToken::new(),
&|_| {},
);
println!(
"common literal, cap 100k: total {:?}, files {}, matches {}, truncated {}",
s.elapsed, s.files, s.matches, s.truncated
);
}
proptest! {
#![proptest_config(ProptestConfig::with_cases(64))]
/// Random patterns, flags and file contents never panic, and every
/// reported match lies inside its preview.
#[test]
fn random_queries_never_panic(
pattern in "\\PC{0,12}",
regex in any::<bool>(),
case_sensitive in any::<bool>(),
whole_word in any::<bool>(),
content in proptest::collection::vec(any::<u8>(), 0..400),
text in "\\PC{0,200}",
) {
let dir = tempfile::tempdir().unwrap();
fs::write(dir.path().join("bytes.txt"), &content).unwrap();
fs::write(dir.path().join("text.php"), &text).unwrap();
let query = Query {
pattern,
regex,
case_sensitive,
whole_word,
include: vec![],
exclude: vec![],
max_results: 50,
};
let Ok(search) = Search::new(&query) else {
return Ok(());
};
let found = Mutex::new(Vec::new());
let summary = run(dir.path(), &search, &Overlay::new(), &CancelToken::new(), &|f| {
found.lock().unwrap().push(f)
});
let found = found.into_inner().unwrap();
prop_assert!(summary.matches <= 50);
for file in found {
for m in file.matches {
prop_assert!(m.length > 0);
let units = m.preview.encode_utf16().count();
prop_assert!(m.preview_column + m.length <= units || m.preview_column <= units);
}
}
}
}