M2.3: find in files backend

crates/search: parallel ignore walk + grep-searcher with the project's
ignore rules (vendor searched as library, node_modules and .git skipped),
literal or regex, case, whole word, include/exclude globs, binary and
> 8 MiB skips, match cap, cancel token, buffer overlay.

Engine: search.find returns {searchId} at once and runs off the lock;
pump_vfs drains results into searchResults (one per file, at most 500
per pump) and searchDone; search.cancel. Testkit: generated search tree.
Tests: library unit and property tests, engine tests, TCP control test.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
danny 2026-10-16 22:03:17 +02:00
parent a6b2fd1300
commit 172693c8dd
21 changed files with 2231 additions and 5 deletions

View file

@ -10,6 +10,8 @@
clippy::indexing_slicing
)]
pub mod tree;
use std::path::{Path, PathBuf};
use std::sync::Mutex;
@ -17,6 +19,8 @@ use lutra_control::{Backend, EventSink, Subscribers};
use lutra_engine::{CommandInfo, Effect, Engine, EngineConfig, EngineError};
use serde_json::Value;
pub use tree::{NEEDLE, SearchTree, generate_search_tree};
/// Engine directories inside `root` (`root/config`, `root/data`), for
/// [`Engine::with_config`] and [`EngineBackend::with_config`] in tests.
pub fn engine_config(root: &Path) -> EngineConfig {

151
crates/testkit/src/tree.rs Normal file
View file

@ -0,0 +1,151 @@
//! Generated project trees for search tests and measurements.
//!
//! [`generate_search_tree`] writes a deterministic PHP project of any size:
//! project sources, templates, scripts and docs, a gitignored `var/`, a
//! gitignored Composer `vendor/` (searched as library code), an excluded
//! `node_modules/` and a binary file. Every [`NEEDLE_EVERY`]th file of each
//! group contains [`NEEDLE`] once; [`SearchTree`] reports how many files of
//! each group have it, so tests know the expected result without searching
//! twice.
use std::fs;
use std::path::Path;
/// A token that occurs only where the generator puts it.
pub const NEEDLE: &str = "lutraNeedle";
/// Every this many files of a group contains [`NEEDLE`].
pub const NEEDLE_EVERY: usize = 50;
/// What [`generate_search_tree`] wrote.
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
pub struct SearchTree {
/// Files written in total, including `.gitignore` and the binary file.
pub files: usize,
/// Searchable files (sources, templates, scripts, docs, vendor) that
/// contain [`NEEDLE`].
pub needle_files: usize,
/// Of those, files under `vendor/`.
pub needle_vendor: usize,
/// Files containing [`NEEDLE`] that a search must skip (ignored,
/// excluded or binary).
pub needle_skipped: usize,
}
/// Writes a project of about `files` files below `root`.
///
/// Shares: 70 % PHP sources in `src/`, 8 % templates, 5 % JavaScript, 5 %
/// Markdown, 6 % ignored PHP in `var/cache/`, 6 % PHP in `vendor/`, plus a
/// few files in `node_modules/` and `bin/data.bin` (a NUL byte, then the
/// needle). Every PHP file contains `function` on several lines, the common
/// literal for measurements.
///
/// # Panics
/// If a file cannot be written (test helper).
pub fn generate_search_tree(root: &Path, files: usize) -> SearchTree {
let mut tree = SearchTree::default();
write(root, ".gitignore", "/var/\n/vendor/\n*.log\n", &mut tree);
let groups: [(&str, usize, Kind); 6] = [
("src", 70, Kind::Php),
("templates", 8, Kind::Twig),
("assets", 5, Kind::Js),
("docs", 5, Kind::Markdown),
("var/cache", 6, Kind::Php),
("vendor/acme/toolkit/src", 6, Kind::Php),
];
for (dir, share, kind) in groups {
let count = files.saturating_mul(share) / 100;
for i in 0..count {
let needle = i % NEEDLE_EVERY == 0;
let rel = format!("{dir}/Module{:03}/{}", i / 100, kind.file_name(i));
write(root, &rel, &kind.content(i, needle), &mut tree);
if !needle {
continue;
}
if dir == "var/cache" {
tree.needle_skipped += 1;
} else {
tree.needle_files += 1;
if dir.starts_with("vendor") {
tree.needle_vendor += 1;
}
}
}
}
for i in 0..3 {
let rel = format!("node_modules/pkg{i}/index.js");
write(root, &rel, &Kind::Js.content(i, true), &mut tree);
tree.needle_skipped += 1;
}
write(root, "debug.log", &format!("{NEEDLE}\n"), &mut tree);
tree.needle_skipped += 1;
let mut binary = b"\x89BIN\x00\x01\x02".to_vec();
binary.extend_from_slice(NEEDLE.as_bytes());
fs::create_dir_all(root.join("bin")).unwrap();
fs::write(root.join("bin/data.bin"), binary).unwrap();
tree.files += 1;
tree.needle_skipped += 1;
tree
}
/// Writes `text` to `root/rel`, creating directories, and counts it.
fn write(root: &Path, rel: &str, text: &str, tree: &mut SearchTree) {
let path = root.join(rel);
if let Some(parent) = path.parent() {
fs::create_dir_all(parent).unwrap();
}
fs::write(path, text).unwrap();
tree.files += 1;
}
/// The kind of a generated file.
#[derive(Clone, Copy)]
enum Kind {
/// A PHP class.
Php,
/// A Twig template.
Twig,
/// A JavaScript module.
Js,
/// A Markdown page.
Markdown,
}
impl Kind {
/// File name of the `i`th file.
fn file_name(self, i: usize) -> String {
match self {
Kind::Php => format!("Service{i}.php"),
Kind::Twig => format!("page{i}.html.twig"),
Kind::Js => format!("widget{i}.js"),
Kind::Markdown => format!("note{i}.md"),
}
}
/// Content of the `i`th file, with [`NEEDLE`] once if `needle`.
fn content(self, i: usize, needle: bool) -> String {
let marker = if needle { NEEDLE } else { "plain" };
match self {
Kind::Php => {
let mut s = format!(
"<?php\n\ndeclare(strict_types=1);\n\nnamespace App\\Module{};\n\nuse App\\Contracts\\Handler;\n\n/**\n * Service number {i}.\n */\nfinal class Service{i} implements Handler\n{{\n private array $items = [];\n\n",
i / 100
);
for m in 0..8 {
s.push_str(&format!(
" public function handle{m}(int $id, string $name): ?string\n {{\n $key = sprintf('%s-%d', $name, $id + {m});\n if (isset($this->items[$key])) {{\n return $this->items[$key];\n }}\n return null;\n }}\n\n"
));
}
s.push_str(&format!(" public const MARKER = '{marker}';\n}}\n"));
s
}
Kind::Twig => format!(
"{{% extends 'base.html.twig' %}}\n{{% block body %}}\n <h1>Page {i}</h1>\n <p class=\"{marker}\">{{{{ content }}}}</p>\n{{% endblock %}}\n"
),
Kind::Js => format!(
"export function widget{i}(el) {{\n el.dataset.kind = '{marker}';\n return el;\n}}\n"
),
Kind::Markdown => format!("# Note {i}\n\nSome text about {marker} and more text.\n"),
}
}
}