crates/search: parallel ignore walk + grep-searcher with the project's
ignore rules (vendor searched as library, node_modules and .git skipped),
literal or regex, case, whole word, include/exclude globs, binary and
> 8 MiB skips, match cap, cancel token, buffer overlay.
Engine: search.find returns {searchId} at once and runs off the lock;
pump_vfs drains results into searchResults (one per file, at most 500
per pump) and searchDone; search.cancel. Testkit: generated search tree.
Tests: library unit and property tests, engine tests, TCP control test.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
195 lines
6.2 KiB
Rust
195 lines
6.2 KiB
Rust
//! Turning a matching line into [`Match`]es: UTF-16 columns and previews.
|
|
|
|
use grep_matcher::Matcher;
|
|
use grep_regex::RegexMatcher;
|
|
|
|
use crate::{MAX_PREVIEW_CHARS, Match};
|
|
|
|
/// Characters shown before the match when a long line is cut to a window.
|
|
const PREVIEW_LEAD: usize = 80;
|
|
|
|
/// Appends the matches of one line to `out`, at most `limit` in total.
|
|
///
|
|
/// # Parameters
|
|
/// - `matcher`: the compiled pattern.
|
|
/// - `line`: 0-based line number.
|
|
/// - `bytes`: the line as read, with or without its line break. Invalid
|
|
/// UTF-8 is replaced by U+FFFD before matching, so positions refer to the
|
|
/// text an editor shows.
|
|
/// - `out`: receives the matches; zero-length matches are skipped.
|
|
/// - `limit`: stop once `out` holds this many matches.
|
|
///
|
|
/// A matcher error (none is expected for an in-memory regex) ends the line
|
|
/// early; the matches found so far are kept.
|
|
pub(crate) fn collect(
|
|
matcher: &RegexMatcher,
|
|
line: usize,
|
|
bytes: &[u8],
|
|
out: &mut Vec<Match>,
|
|
limit: usize,
|
|
) {
|
|
let bytes = trim_line_break(bytes);
|
|
let text = String::from_utf8_lossy(bytes);
|
|
let mut spans = Vec::new();
|
|
let _matcher_error = matcher.find_iter(text.as_bytes(), |m| {
|
|
if m.start() < m.end() {
|
|
spans.push((m.start(), m.end()));
|
|
}
|
|
out.len().saturating_add(spans.len()) < limit
|
|
});
|
|
if spans.is_empty() {
|
|
return;
|
|
}
|
|
let window = Window::new(&text);
|
|
let mut cursor = Cursor::default();
|
|
for (start, end) in spans {
|
|
let Some(matched) = text.get(start..end) else {
|
|
continue;
|
|
};
|
|
if !cursor.advance(&text, start) {
|
|
continue;
|
|
}
|
|
let (preview, preview_column) = window.preview(&text, &cursor, start);
|
|
out.push(Match {
|
|
line,
|
|
column: cursor.utf16,
|
|
length: utf16_len(matched),
|
|
preview,
|
|
preview_column,
|
|
});
|
|
}
|
|
}
|
|
|
|
/// `bytes` without a trailing `\n` or `\r\n`.
|
|
fn trim_line_break(bytes: &[u8]) -> &[u8] {
|
|
let bytes = bytes.strip_suffix(b"\n").unwrap_or(bytes);
|
|
bytes.strip_suffix(b"\r").unwrap_or(bytes)
|
|
}
|
|
|
|
/// Number of UTF-16 code units of `s`.
|
|
fn utf16_len(s: &str) -> usize {
|
|
s.chars().map(char::len_utf16).sum()
|
|
}
|
|
|
|
/// A position in the line, counted three ways; only moves forward.
|
|
#[derive(Default)]
|
|
struct Cursor {
|
|
/// Byte offset.
|
|
byte: usize,
|
|
/// Characters before `byte`.
|
|
chars: usize,
|
|
/// UTF-16 code units before `byte`.
|
|
utf16: usize,
|
|
}
|
|
|
|
impl Cursor {
|
|
/// Moves to byte offset `to`; false if `to` lies behind the cursor or
|
|
/// not on a character boundary.
|
|
fn advance(&mut self, text: &str, to: usize) -> bool {
|
|
let Some(between) = text.get(self.byte..to) else {
|
|
return false;
|
|
};
|
|
for c in between.chars() {
|
|
self.chars = self.chars.saturating_add(1);
|
|
self.utf16 = self.utf16.saturating_add(c.len_utf16());
|
|
}
|
|
self.byte = to;
|
|
true
|
|
}
|
|
}
|
|
|
|
/// How previews of one line are cut.
|
|
enum Window {
|
|
/// The line is short enough to be the preview of every match.
|
|
Whole,
|
|
/// The line is longer than [`MAX_PREVIEW_CHARS`]: the byte offset of
|
|
/// every character, plus the text length at the end.
|
|
Cut(Vec<usize>),
|
|
}
|
|
|
|
impl Window {
|
|
/// Prepares previews for `text`.
|
|
fn new(text: &str) -> Self {
|
|
if text.len() <= MAX_PREVIEW_CHARS || text.chars().count() <= MAX_PREVIEW_CHARS {
|
|
return Self::Whole;
|
|
}
|
|
let mut offsets: Vec<usize> = text.char_indices().map(|(i, _)| i).collect();
|
|
offsets.push(text.len());
|
|
Self::Cut(offsets)
|
|
}
|
|
|
|
/// The preview for a match at `cursor` (byte offset `start`), and the
|
|
/// match's UTF-16 column inside it.
|
|
fn preview(&self, text: &str, cursor: &Cursor, start: usize) -> (String, usize) {
|
|
let Self::Cut(offsets) = self else {
|
|
return (text.to_string(), cursor.utf16);
|
|
};
|
|
let total = offsets.len().saturating_sub(1);
|
|
let first = cursor
|
|
.chars
|
|
.saturating_sub(PREVIEW_LEAD)
|
|
.min(total.saturating_sub(MAX_PREVIEW_CHARS));
|
|
let last = first.saturating_add(MAX_PREVIEW_CHARS).min(total);
|
|
let from = offsets.get(first).copied().unwrap_or(0);
|
|
let to = offsets.get(last).copied().unwrap_or(text.len());
|
|
let preview = text.get(from..to).unwrap_or_default().to_string();
|
|
let column = text.get(from..start).map(utf16_len).unwrap_or(0);
|
|
(preview, column)
|
|
}
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use grep_regex::RegexMatcherBuilder;
|
|
|
|
use super::*;
|
|
|
|
fn matcher(pattern: &str) -> RegexMatcher {
|
|
RegexMatcherBuilder::new()
|
|
.fixed_strings(true)
|
|
.line_terminator(Some(b'\n'))
|
|
.build(pattern)
|
|
.unwrap()
|
|
}
|
|
|
|
#[test]
|
|
fn columns_count_utf16_units() {
|
|
let mut out = Vec::new();
|
|
collect(&matcher("x"), 4, "ä😀x and x\r\n".as_bytes(), &mut out, 10);
|
|
assert_eq!(out.len(), 2);
|
|
assert_eq!((out[0].line, out[0].column, out[0].length), (4, 3, 1));
|
|
assert_eq!(out[0].preview, "ä😀x and x");
|
|
assert_eq!(out[0].preview_column, 3);
|
|
assert_eq!(out[1].column, 9);
|
|
}
|
|
|
|
#[test]
|
|
fn long_lines_are_cut_around_the_match() {
|
|
let line = format!("{}needle{}", "a".repeat(1000), "b".repeat(1000));
|
|
let mut out = Vec::new();
|
|
collect(&matcher("needle"), 0, line.as_bytes(), &mut out, 10);
|
|
let m = &out[0];
|
|
assert_eq!(m.column, 1000);
|
|
assert_eq!(m.preview.chars().count(), MAX_PREVIEW_CHARS);
|
|
assert_eq!(
|
|
m.preview.get(m.preview_column..m.preview_column + 6),
|
|
Some("needle")
|
|
);
|
|
assert_eq!(m.preview_column, PREVIEW_LEAD);
|
|
|
|
let tail = format!("{}needle", "a".repeat(1000));
|
|
out.clear();
|
|
collect(&matcher("needle"), 0, tail.as_bytes(), &mut out, 10);
|
|
assert!(out[0].preview.ends_with("needle"));
|
|
assert_eq!(out[0].preview.chars().count(), MAX_PREVIEW_CHARS);
|
|
}
|
|
|
|
#[test]
|
|
fn invalid_utf8_and_limits() {
|
|
let mut out = Vec::new();
|
|
collect(&matcher("ab"), 0, b"\xffab ab ab\n", &mut out, 2);
|
|
assert_eq!(out.len(), 2);
|
|
assert_eq!(out[0].column, 1);
|
|
assert_eq!(out[0].preview, "\u{fffd}ab ab ab");
|
|
}
|
|
}
|