M1.1: text buffer with UTF-16 positions, edit batches and line endings
lutra-text gets Buffer on a ropey rope: byte/char/UTF-16/line counts, byte offset <-> (line, column) in UTF-16 code units or chars, absolute UTF-16 offsets, atomic Edit batches in LSP TextEdit[] semantics, and \n / \r\n / \r detection with a keep-the-file's-ending save policy. All conversions return TextError instead of panicking. ropey 1.6.1 (MIT) without unicode_lines, so only the LSP/CodeMirror line breaks split lines. proptest moves into workspace deps. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
faadb05eb3
commit
5dac918b14
12 changed files with 1278 additions and 3 deletions
241
crates/text/tests/proptests.rs
Normal file
241
crates/text/tests/proptests.rs
Normal file
|
|
@ -0,0 +1,241 @@
|
|||
//! Property tests: conversions round-trip on random Unicode and random
|
||||
//! edits, valid or not, never panic and match a plain `String` model.
|
||||
|
||||
#![allow(
|
||||
clippy::unwrap_used,
|
||||
clippy::expect_used,
|
||||
clippy::indexing_slicing,
|
||||
clippy::string_slice
|
||||
)]
|
||||
|
||||
use lutra_text::{Buffer, ColumnUnit, Edit, LineEnding, Position, TextError};
|
||||
use proptest::prelude::*;
|
||||
|
||||
const UNITS: [ColumnUnit; 2] = [ColumnUnit::Utf16, ColumnUnit::Char];
|
||||
|
||||
/// Text built from pieces that stress positions: ASCII, all three line
|
||||
/// breaks, emoji (astral, ZWJ sequences, flags), CJK, combining marks,
|
||||
/// Unicode separators that are not line breaks, and the highest code point.
|
||||
fn tricky_text() -> impl Strategy<Value = String> {
|
||||
let piece = prop_oneof![
|
||||
Just("a"),
|
||||
Just("Z"),
|
||||
Just(" "),
|
||||
Just("\t"),
|
||||
Just("\n"),
|
||||
Just("\r\n"),
|
||||
Just("\r"),
|
||||
Just("😀"),
|
||||
Just("👨\u{200d}👩\u{200d}👧"),
|
||||
Just("🇩🇪"),
|
||||
Just("漢"),
|
||||
Just("字"),
|
||||
Just("e\u{301}"),
|
||||
Just("\u{301}"),
|
||||
Just("\u{2028}"),
|
||||
Just("\u{85}"),
|
||||
Just("\u{10ffff}"),
|
||||
Just("é"),
|
||||
];
|
||||
prop::collection::vec(piece, 0..40).prop_map(|pieces| pieces.concat())
|
||||
}
|
||||
|
||||
/// Tricky text or fully random strings.
|
||||
fn any_text() -> impl Strategy<Value = String> {
|
||||
prop_oneof![3 => tricky_text(), 1 => any::<String>()]
|
||||
}
|
||||
|
||||
/// Edit ranges anywhere, including past the end and reversed.
|
||||
fn any_edit(max: usize) -> impl Strategy<Value = Edit> {
|
||||
(0..=max + 3, 0..=max + 3, tricky_text())
|
||||
.prop_map(|(start, end, text)| Edit::replace(start..end, text))
|
||||
}
|
||||
|
||||
/// The line breaks of `text` as byte ranges, CRLF as one break.
|
||||
fn line_breaks(text: &str) -> Vec<std::ops::Range<usize>> {
|
||||
let bytes = text.as_bytes();
|
||||
let mut breaks = Vec::new();
|
||||
let mut i = 0;
|
||||
while i < bytes.len() {
|
||||
match bytes[i] {
|
||||
b'\r' if bytes.get(i + 1) == Some(&b'\n') => {
|
||||
breaks.push(i..i + 2);
|
||||
i += 2;
|
||||
}
|
||||
b'\r' | b'\n' => {
|
||||
breaks.push(i..i + 1);
|
||||
i += 1;
|
||||
}
|
||||
_ => i += 1,
|
||||
}
|
||||
}
|
||||
breaks
|
||||
}
|
||||
|
||||
/// Reference position of a byte offset, computed with `str` methods.
|
||||
fn model_position(text: &str, offset: usize, unit: ColumnUnit) -> Option<Position> {
|
||||
if !text.is_char_boundary(offset) {
|
||||
return None;
|
||||
}
|
||||
let breaks = line_breaks(text);
|
||||
if breaks.iter().any(|b| b.start < offset && offset < b.end) {
|
||||
return None;
|
||||
}
|
||||
let line = breaks.iter().filter(|b| b.end <= offset).count();
|
||||
let line_start = breaks
|
||||
.iter()
|
||||
.filter(|b| b.end <= offset)
|
||||
.map(|b| b.end)
|
||||
.next_back()
|
||||
.unwrap_or(0);
|
||||
let prefix = &text[line_start..offset];
|
||||
let column = match unit {
|
||||
ColumnUnit::Utf16 => prefix.encode_utf16().count(),
|
||||
ColumnUnit::Char => prefix.chars().count(),
|
||||
};
|
||||
Some(Position::new(line, column))
|
||||
}
|
||||
|
||||
/// Applies edits to a `String` with the documented batch semantics, or
|
||||
/// `None` where the buffer must reject the batch.
|
||||
fn model_apply(text: &str, edits: &[Edit]) -> Option<String> {
|
||||
let mut sorted: Vec<&Edit> = edits.iter().collect();
|
||||
for e in &sorted {
|
||||
if e.range.start > e.range.end
|
||||
|| !text.is_char_boundary(e.range.start)
|
||||
|| !text.is_char_boundary(e.range.end)
|
||||
{
|
||||
return None;
|
||||
}
|
||||
}
|
||||
sorted.sort_by_key(|e| (e.range.start, e.range.end));
|
||||
if sorted.windows(2).any(|w| w[0].range.end > w[1].range.start) {
|
||||
return None;
|
||||
}
|
||||
let mut out = String::new();
|
||||
let mut at = 0;
|
||||
for e in sorted {
|
||||
out.push_str(&text[at..e.range.start]);
|
||||
out.push_str(&e.text);
|
||||
at = e.range.end;
|
||||
}
|
||||
out.push_str(&text[at..]);
|
||||
Some(out)
|
||||
}
|
||||
|
||||
proptest! {
|
||||
#![proptest_config(ProptestConfig::with_cases(512))]
|
||||
|
||||
#[test]
|
||||
fn counts_match_the_string(text in any_text()) {
|
||||
let b = Buffer::new(&text);
|
||||
prop_assert_eq!(b.text(), text.clone());
|
||||
prop_assert_eq!(b.len_bytes(), text.len());
|
||||
prop_assert_eq!(b.len_chars(), text.chars().count());
|
||||
prop_assert_eq!(b.len_utf16(), text.encode_utf16().count());
|
||||
prop_assert_eq!(b.line_count(), line_breaks(&text).len() + 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn every_byte_offset_converts_like_the_model_and_round_trips(text in any_text()) {
|
||||
let b = Buffer::new(&text);
|
||||
for offset in 0..=text.len() + 2 {
|
||||
for unit in UNITS {
|
||||
let got = b.offset_to_position(offset, unit);
|
||||
prop_assert_eq!(got.as_ref().ok().copied(), model_position(&text, offset, unit));
|
||||
if let Ok(position) = got {
|
||||
prop_assert_eq!(b.position_to_offset(position, unit), Ok(offset));
|
||||
}
|
||||
}
|
||||
let utf16 = b.offset_to_utf16(offset);
|
||||
prop_assert_eq!(utf16.is_ok(), text.is_char_boundary(offset));
|
||||
if let Ok(cu) = utf16 {
|
||||
prop_assert_eq!(cu, text[..offset].encode_utf16().count());
|
||||
prop_assert_eq!(b.utf16_to_offset(cu), Ok(offset));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn random_positions_never_panic_and_round_trip(
|
||||
text in any_text(),
|
||||
line in 0usize..12,
|
||||
column in 0usize..48,
|
||||
) {
|
||||
let b = Buffer::new(&text);
|
||||
for unit in UNITS {
|
||||
if let Ok(offset) = b.position_to_offset(Position::new(line, column), unit) {
|
||||
prop_assert_eq!(
|
||||
b.offset_to_position(offset, unit),
|
||||
Ok(Position::new(line, column))
|
||||
);
|
||||
}
|
||||
}
|
||||
for cu in 0..=b.len_utf16() + 2 {
|
||||
if let Ok(offset) = b.utf16_to_offset(cu) {
|
||||
prop_assert_eq!(b.offset_to_utf16(offset), Ok(cu));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn random_edit_batches_never_panic_and_match_the_model(
|
||||
text in tricky_text(),
|
||||
edits in prop::collection::vec(any_edit(160), 0..5),
|
||||
) {
|
||||
let mut b = Buffer::new(&text);
|
||||
let expected = model_apply(&text, &edits);
|
||||
let result = b.apply(&edits);
|
||||
prop_assert_eq!(result.is_ok(), expected.is_some(), "{:?}", result);
|
||||
prop_assert_eq!(b.text(), expected.unwrap_or(text));
|
||||
prop_assert_eq!(b.len_bytes(), b.text().len());
|
||||
prop_assert_eq!(b.line_count(), line_breaks(&b.text()).len() + 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sequences_of_single_edits_stay_consistent(
|
||||
text in tricky_text(),
|
||||
edits in prop::collection::vec(any_edit(80), 0..20),
|
||||
) {
|
||||
let mut b = Buffer::new(&text);
|
||||
let mut model = text;
|
||||
for edit in edits {
|
||||
let expected = model_apply(&model, std::slice::from_ref(&edit));
|
||||
let result = b.apply(std::slice::from_ref(&edit));
|
||||
prop_assert_eq!(result.is_ok(), expected.is_some());
|
||||
if let Some(next) = expected {
|
||||
model = next;
|
||||
}
|
||||
prop_assert_eq!(b.text(), model.clone());
|
||||
}
|
||||
for line in 0..b.line_count() {
|
||||
let range = b.line_range(line).unwrap();
|
||||
for unit in UNITS {
|
||||
let start = b.offset_to_position(range.start, unit).unwrap();
|
||||
prop_assert_eq!(start, Position::new(line, 0));
|
||||
let end = b.offset_to_position(range.end, unit).unwrap();
|
||||
prop_assert_eq!(b.position_to_offset(end, unit), Ok(range.end));
|
||||
let past_end = Position::new(line, end.column + 1);
|
||||
let is_out_of_range = matches!(
|
||||
b.position_to_offset(past_end, unit),
|
||||
Err(TextError::ColumnOutOfRange { .. })
|
||||
);
|
||||
prop_assert!(is_out_of_range);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn save_text_uses_only_the_buffers_line_ending(text in tricky_text()) {
|
||||
let b = Buffer::new(&text);
|
||||
let ending = b.line_ending();
|
||||
prop_assert_eq!(Some(ending), LineEnding::detect(&text).or(Some(LineEnding::Lf)));
|
||||
let saved = b.save_text();
|
||||
let breaks = line_breaks(&saved);
|
||||
prop_assert_eq!(breaks.len(), line_breaks(&text).len());
|
||||
for range in breaks {
|
||||
prop_assert_eq!(&saved[range], ending.as_str());
|
||||
}
|
||||
prop_assert_eq!(Buffer::new(&saved).text(), saved);
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue