M2.3: find in files backend
crates/search: parallel ignore walk + grep-searcher with the project's
ignore rules (vendor searched as library, node_modules and .git skipped),
literal or regex, case, whole word, include/exclude globs, binary and
> 8 MiB skips, match cap, cancel token, buffer overlay.
Engine: search.find returns {searchId} at once and runs off the lock;
pump_vfs drains results into searchResults (one per file, at most 500
per pump) and searchDone; search.cancel. Testkit: generated search tree.
Tests: library unit and property tests, engine tests, TCP control test.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
a6b2fd1300
commit
172693c8dd
21 changed files with 2231 additions and 5 deletions
|
|
@ -10,6 +10,8 @@
|
|||
clippy::indexing_slicing
|
||||
)]
|
||||
|
||||
pub mod tree;
|
||||
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::Mutex;
|
||||
|
||||
|
|
@ -17,6 +19,8 @@ use lutra_control::{Backend, EventSink, Subscribers};
|
|||
use lutra_engine::{CommandInfo, Effect, Engine, EngineConfig, EngineError};
|
||||
use serde_json::Value;
|
||||
|
||||
pub use tree::{NEEDLE, SearchTree, generate_search_tree};
|
||||
|
||||
/// Engine directories inside `root` (`root/config`, `root/data`), for
|
||||
/// [`Engine::with_config`] and [`EngineBackend::with_config`] in tests.
|
||||
pub fn engine_config(root: &Path) -> EngineConfig {
|
||||
|
|
|
|||
151
crates/testkit/src/tree.rs
Normal file
151
crates/testkit/src/tree.rs
Normal file
|
|
@ -0,0 +1,151 @@
|
|||
//! Generated project trees for search tests and measurements.
|
||||
//!
|
||||
//! [`generate_search_tree`] writes a deterministic PHP project of any size:
|
||||
//! project sources, templates, scripts and docs, a gitignored `var/`, a
|
||||
//! gitignored Composer `vendor/` (searched as library code), an excluded
|
||||
//! `node_modules/` and a binary file. Every [`NEEDLE_EVERY`]th file of each
|
||||
//! group contains [`NEEDLE`] once; [`SearchTree`] reports how many files of
|
||||
//! each group have it, so tests know the expected result without searching
|
||||
//! twice.
|
||||
|
||||
use std::fs;
|
||||
use std::path::Path;
|
||||
|
||||
/// A token that occurs only where the generator puts it.
|
||||
pub const NEEDLE: &str = "lutraNeedle";
|
||||
|
||||
/// Every this many files of a group contains [`NEEDLE`].
|
||||
pub const NEEDLE_EVERY: usize = 50;
|
||||
|
||||
/// What [`generate_search_tree`] wrote.
|
||||
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
|
||||
pub struct SearchTree {
|
||||
/// Files written in total, including `.gitignore` and the binary file.
|
||||
pub files: usize,
|
||||
/// Searchable files (sources, templates, scripts, docs, vendor) that
|
||||
/// contain [`NEEDLE`].
|
||||
pub needle_files: usize,
|
||||
/// Of those, files under `vendor/`.
|
||||
pub needle_vendor: usize,
|
||||
/// Files containing [`NEEDLE`] that a search must skip (ignored,
|
||||
/// excluded or binary).
|
||||
pub needle_skipped: usize,
|
||||
}
|
||||
|
||||
/// Writes a project of about `files` files below `root`.
|
||||
///
|
||||
/// Shares: 70 % PHP sources in `src/`, 8 % templates, 5 % JavaScript, 5 %
|
||||
/// Markdown, 6 % ignored PHP in `var/cache/`, 6 % PHP in `vendor/`, plus a
|
||||
/// few files in `node_modules/` and `bin/data.bin` (a NUL byte, then the
|
||||
/// needle). Every PHP file contains `function` on several lines, the common
|
||||
/// literal for measurements.
|
||||
///
|
||||
/// # Panics
|
||||
/// If a file cannot be written (test helper).
|
||||
pub fn generate_search_tree(root: &Path, files: usize) -> SearchTree {
|
||||
let mut tree = SearchTree::default();
|
||||
write(root, ".gitignore", "/var/\n/vendor/\n*.log\n", &mut tree);
|
||||
let groups: [(&str, usize, Kind); 6] = [
|
||||
("src", 70, Kind::Php),
|
||||
("templates", 8, Kind::Twig),
|
||||
("assets", 5, Kind::Js),
|
||||
("docs", 5, Kind::Markdown),
|
||||
("var/cache", 6, Kind::Php),
|
||||
("vendor/acme/toolkit/src", 6, Kind::Php),
|
||||
];
|
||||
for (dir, share, kind) in groups {
|
||||
let count = files.saturating_mul(share) / 100;
|
||||
for i in 0..count {
|
||||
let needle = i % NEEDLE_EVERY == 0;
|
||||
let rel = format!("{dir}/Module{:03}/{}", i / 100, kind.file_name(i));
|
||||
write(root, &rel, &kind.content(i, needle), &mut tree);
|
||||
if !needle {
|
||||
continue;
|
||||
}
|
||||
if dir == "var/cache" {
|
||||
tree.needle_skipped += 1;
|
||||
} else {
|
||||
tree.needle_files += 1;
|
||||
if dir.starts_with("vendor") {
|
||||
tree.needle_vendor += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
for i in 0..3 {
|
||||
let rel = format!("node_modules/pkg{i}/index.js");
|
||||
write(root, &rel, &Kind::Js.content(i, true), &mut tree);
|
||||
tree.needle_skipped += 1;
|
||||
}
|
||||
write(root, "debug.log", &format!("{NEEDLE}\n"), &mut tree);
|
||||
tree.needle_skipped += 1;
|
||||
let mut binary = b"\x89BIN\x00\x01\x02".to_vec();
|
||||
binary.extend_from_slice(NEEDLE.as_bytes());
|
||||
fs::create_dir_all(root.join("bin")).unwrap();
|
||||
fs::write(root.join("bin/data.bin"), binary).unwrap();
|
||||
tree.files += 1;
|
||||
tree.needle_skipped += 1;
|
||||
tree
|
||||
}
|
||||
|
||||
/// Writes `text` to `root/rel`, creating directories, and counts it.
|
||||
fn write(root: &Path, rel: &str, text: &str, tree: &mut SearchTree) {
|
||||
let path = root.join(rel);
|
||||
if let Some(parent) = path.parent() {
|
||||
fs::create_dir_all(parent).unwrap();
|
||||
}
|
||||
fs::write(path, text).unwrap();
|
||||
tree.files += 1;
|
||||
}
|
||||
|
||||
/// The kind of a generated file.
|
||||
#[derive(Clone, Copy)]
|
||||
enum Kind {
|
||||
/// A PHP class.
|
||||
Php,
|
||||
/// A Twig template.
|
||||
Twig,
|
||||
/// A JavaScript module.
|
||||
Js,
|
||||
/// A Markdown page.
|
||||
Markdown,
|
||||
}
|
||||
|
||||
impl Kind {
|
||||
/// File name of the `i`th file.
|
||||
fn file_name(self, i: usize) -> String {
|
||||
match self {
|
||||
Kind::Php => format!("Service{i}.php"),
|
||||
Kind::Twig => format!("page{i}.html.twig"),
|
||||
Kind::Js => format!("widget{i}.js"),
|
||||
Kind::Markdown => format!("note{i}.md"),
|
||||
}
|
||||
}
|
||||
|
||||
/// Content of the `i`th file, with [`NEEDLE`] once if `needle`.
|
||||
fn content(self, i: usize, needle: bool) -> String {
|
||||
let marker = if needle { NEEDLE } else { "plain" };
|
||||
match self {
|
||||
Kind::Php => {
|
||||
let mut s = format!(
|
||||
"<?php\n\ndeclare(strict_types=1);\n\nnamespace App\\Module{};\n\nuse App\\Contracts\\Handler;\n\n/**\n * Service number {i}.\n */\nfinal class Service{i} implements Handler\n{{\n private array $items = [];\n\n",
|
||||
i / 100
|
||||
);
|
||||
for m in 0..8 {
|
||||
s.push_str(&format!(
|
||||
" public function handle{m}(int $id, string $name): ?string\n {{\n $key = sprintf('%s-%d', $name, $id + {m});\n if (isset($this->items[$key])) {{\n return $this->items[$key];\n }}\n return null;\n }}\n\n"
|
||||
));
|
||||
}
|
||||
s.push_str(&format!(" public const MARKER = '{marker}';\n}}\n"));
|
||||
s
|
||||
}
|
||||
Kind::Twig => format!(
|
||||
"{{% extends 'base.html.twig' %}}\n{{% block body %}}\n <h1>Page {i}</h1>\n <p class=\"{marker}\">{{{{ content }}}}</p>\n{{% endblock %}}\n"
|
||||
),
|
||||
Kind::Js => format!(
|
||||
"export function widget{i}(el) {{\n el.dataset.kind = '{marker}';\n return el;\n}}\n"
|
||||
),
|
||||
Kind::Markdown => format!("# Note {i}\n\nSome text about {marker} and more text.\n"),
|
||||
}
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue