From 5dac918b1418c5253c3089cff41fdd6f390cd799 Mon Sep 17 00:00:00 2001 From: danny Date: Wed, 14 Oct 2026 19:20:18 +0200 Subject: [PATCH] M1.1: text buffer with UTF-16 positions, edit batches and line endings lutra-text gets Buffer on a ropey rope: byte/char/UTF-16/line counts, byte offset <-> (line, column) in UTF-16 code units or chars, absolute UTF-16 offsets, atomic Edit batches in LSP TextEdit[] semantics, and \n / \r\n / \r detection with a keep-the-file's-ending save policy. All conversions return TextError instead of panicking. ropey 1.6.1 (MIT) without unicode_lines, so only the LSP/CodeMirror line breaks split lines. proptest moves into workspace deps. Co-Authored-By: Claude Opus 5.5 --- Cargo.lock | 20 ++ Cargo.toml | 2 + crates/text/Cargo.toml | 4 + crates/text/src/buffer.rs | 573 +++++++++++++++++++++++++++++++++ crates/text/src/edit.rs | 36 +++ crates/text/src/error.rs | 151 +++++++++ crates/text/src/lib.rs | 19 ++ crates/text/src/line_ending.rs | 178 ++++++++++ crates/text/src/position.rs | 39 +++ crates/text/tests/proptests.rs | 241 ++++++++++++++ docs/gaps.md | 5 +- docs/target-app-parity.md | 13 +- 12 files changed, 1278 insertions(+), 3 deletions(-) create mode 100644 crates/text/src/buffer.rs create mode 100644 crates/text/src/edit.rs create mode 100644 crates/text/src/error.rs create mode 100644 crates/text/src/line_ending.rs create mode 100644 crates/text/src/position.rs create mode 100644 crates/text/tests/proptests.rs diff --git a/Cargo.lock b/Cargo.lock index a08c740..b50ec71 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2453,6 +2453,10 @@ dependencies = [ [[package]] name = "lutra-text" version = "0.0.0" +dependencies = [ + "proptest", + "ropey", +] [[package]] name = "lutra-vfs" @@ -3521,6 +3525,16 @@ dependencies = [ "web-sys", ] +[[package]] +name = "ropey" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93411e420bcd1a75ddd1dc3caf18c23155eda2c090631a85af21ba19e97093b5" +dependencies = [ + "smallvec", + "str_indices", +] + [[package]] name = "rustc-hash" version = "2.1.3" @@ -3961,6 +3975,12 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" +[[package]] +name = "str_indices" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d08889ec5408683408db66ad89e0e1f93dff55c73a4ccc71c427d5b277ee47e6" + [[package]] name = "string_cache" version = "0.9.0" diff --git a/Cargo.toml b/Cargo.toml index a3abfb3..51f8e8b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -23,6 +23,8 @@ lutra-engine = { path = "crates/engine" } lutra-control = { path = "crates/control" } lutra-mcp = { path = "crates/mcp" } lutra-testkit = { path = "crates/testkit" } +ropey = { version = "1.6.1", default-features = false, features = ["cr_lines", "simd"] } +proptest = "1.12.0" [workspace.lints.rust] missing_docs = "deny" diff --git a/crates/text/Cargo.toml b/crates/text/Cargo.toml index d145215..bbdbb7e 100644 --- a/crates/text/Cargo.toml +++ b/crates/text/Cargo.toml @@ -12,6 +12,10 @@ publish.workspace = true layer = 0 [dependencies] +ropey.workspace = true + +[dev-dependencies] +proptest.workspace = true [lints] workspace = true diff --git a/crates/text/src/buffer.rs b/crates/text/src/buffer.rs new file mode 100644 index 0000000..d08bc90 --- /dev/null +++ b/crates/text/src/buffer.rs @@ -0,0 +1,573 @@ +//! The rope-backed text buffer. + +use std::ops::Range; + +use ropey::{Rope, RopeSlice}; + +use crate::{ColumnUnit, Edit, LineEnding, Position, TextError}; + +/// An editable text held in a rope, with the line ending of its file. +/// +/// Offsets are byte offsets into the UTF-8 text. Every conversion checks +/// its input and returns a [`TextError`] for offsets past the end, inside a +/// character or inside a CRLF line break, so offsets from untrusted sources +/// can be passed straight in. Line breaks are `\n`, `\r\n` and a lone `\r` +/// (see [`crate::line_ending`]); the line count is the number of line +/// breaks plus one, so a text ending in a line break has an empty last line. +#[derive(Debug, Clone, Default)] +pub struct Buffer { + /// The text. + rope: Rope, + /// The line ending used when saving. + line_ending: LineEnding, +} + +impl Buffer { + /// Builds a buffer from `text`, taking the line ending detected in it or + /// [`LineEnding::Lf`] if the text has no line break. + pub fn new(text: &str) -> Self { + Self { + rope: Rope::from_str(text), + line_ending: LineEnding::detect(text).unwrap_or_default(), + } + } + + /// The whole text as a string. + pub fn text(&self) -> String { + self.rope.to_string() + } + + /// The text in the byte `range`. + /// + /// # Errors + /// Fails when the range is reversed, reaches past the end or does not + /// lie on character boundaries. + pub fn slice(&self, range: Range) -> Result { + let chars = self.char_range(&range)?; + Ok(self + .rope + .get_slice(chars) + .map(String::from) + .unwrap_or_default()) + } + + /// Length of the text in bytes. + pub fn len_bytes(&self) -> usize { + self.rope.len_bytes() + } + + /// Length of the text in chars (Unicode scalar values). + pub fn len_chars(&self) -> usize { + self.rope.len_chars() + } + + /// Length of the text in UTF-16 code units. + pub fn len_utf16(&self) -> usize { + self.rope.len_utf16_cu() + } + + /// Number of lines: the number of line breaks plus one. + pub fn line_count(&self) -> usize { + self.rope.len_lines() + } + + /// Whether the text is empty. + pub fn is_empty(&self) -> bool { + self.rope.len_bytes() == 0 + } + + /// The line ending the text is saved with. + pub fn line_ending(&self) -> LineEnding { + self.line_ending + } + + /// Changes the line ending the text is saved with. + pub fn set_line_ending(&mut self, line_ending: LineEnding) { + self.line_ending = line_ending; + } + + /// The text to write to disk: every line break replaced by the buffer's + /// line ending, so the file keeps its ending. + pub fn save_text(&self) -> String { + self.line_ending.normalize_chunks(self.rope.chunks()) + } + + /// The byte range of `line` without its line break. + /// + /// # Errors + /// [`TextError::LineOutOfRange`] when the line does not exist. + pub fn line_range(&self, line: usize) -> Result, TextError> { + let slice = self.rope.get_line(line).ok_or(TextError::LineOutOfRange { + line, + line_count: self.line_count(), + })?; + let start = self.rope.try_line_to_byte(line)?; + let content = slice.len_bytes().saturating_sub(line_break_len(&slice)); + Ok(start..start.saturating_add(content)) + } + + /// The text of `line` without its line break. + /// + /// # Errors + /// [`TextError::LineOutOfRange`] when the line does not exist. + pub fn line_text(&self, line: usize) -> Result { + let range = self.line_range(line)?; + self.slice(range) + } + + /// Converts a byte offset to a line and a column counted in `unit`. + /// + /// # Errors + /// Fails when the offset is past the end, inside a character or between + /// the `\r` and `\n` of a CRLF line break. + pub fn offset_to_position( + &self, + offset: usize, + unit: ColumnUnit, + ) -> Result { + let char_idx = self.offset_to_char(offset)?; + let line = self.rope.try_char_to_line(char_idx)?; + let range = self.line_range(line)?; + if offset > range.end { + return Err(TextError::InsideLineBreak { offset }); + } + let line_start = self.rope.try_byte_to_char(range.start)?; + let column = match unit { + ColumnUnit::Char => char_idx.saturating_sub(line_start), + ColumnUnit::Utf16 => self + .rope + .try_char_to_utf16_cu(char_idx)? + .saturating_sub(self.rope.try_char_to_utf16_cu(line_start)?), + }; + Ok(Position { line, column }) + } + + /// Converts a line and a column counted in `unit` to a byte offset. + /// + /// # Errors + /// Fails when the line does not exist, the column is past the end of + /// the line (line break excluded) or, in UTF-16, falls inside a + /// surrogate pair. + pub fn position_to_offset( + &self, + position: Position, + unit: ColumnUnit, + ) -> Result { + let Position { line, column } = position; + let range = self.line_range(line)?; + let start_char = self.rope.try_byte_to_char(range.start)?; + let end_char = self.rope.try_byte_to_char(range.end)?; + let char_idx = match unit { + ColumnUnit::Char => { + let line_len = end_char.saturating_sub(start_char); + if column > line_len { + return Err(TextError::ColumnOutOfRange { + line, + column, + line_len, + }); + } + start_char.saturating_add(column) + } + ColumnUnit::Utf16 => { + let start_cu = self.rope.try_char_to_utf16_cu(start_char)?; + let end_cu = self.rope.try_char_to_utf16_cu(end_char)?; + let line_len = end_cu.saturating_sub(start_cu); + if column > line_len { + return Err(TextError::ColumnOutOfRange { + line, + column, + line_len, + }); + } + let target = start_cu.saturating_add(column); + let char_idx = self.rope.try_utf16_cu_to_char(target)?; + if self.rope.try_char_to_utf16_cu(char_idx)? != target { + return Err(TextError::ColumnInsideChar { line, column }); + } + char_idx + } + }; + Ok(self.rope.try_char_to_byte(char_idx)?) + } + + /// Converts a byte offset to an offset in UTF-16 code units from the + /// start of the text. Line breaks count as stored (`\r\n` is two). + /// + /// # Errors + /// Fails when the offset is past the end or inside a character. + pub fn offset_to_utf16(&self, offset: usize) -> Result { + let char_idx = self.offset_to_char(offset)?; + Ok(self.rope.try_char_to_utf16_cu(char_idx)?) + } + + /// Converts an offset in UTF-16 code units from the start of the text to + /// a byte offset. + /// + /// # Errors + /// Fails when the offset is past the end or inside a surrogate pair. + pub fn utf16_to_offset(&self, offset: usize) -> Result { + let len = self.len_utf16(); + if offset > len { + return Err(TextError::Utf16OutOfRange { offset, len }); + } + let char_idx = self.rope.try_utf16_cu_to_char(offset)?; + if self.rope.try_char_to_utf16_cu(char_idx)? != offset { + return Err(TextError::Utf16InsideChar { offset }); + } + Ok(self.rope.try_char_to_byte(char_idx)?) + } + + /// Applies a batch of edits atomically. + /// + /// All ranges refer to the text before the call, as in an LSP + /// `TextEdit[]`, and may come in any order. Insertions at the same + /// offset land in the order given, and an insertion at the start of a + /// replaced range lands before the replacement. + /// + /// # Errors + /// Rejects the whole batch, leaving the buffer unchanged, when a range + /// is reversed, reaches past the end, does not lie on character + /// boundaries, or overlaps another edit of the batch. + pub fn apply(&mut self, edits: &[Edit]) -> Result<(), TextError> { + let mut resolved = Vec::with_capacity(edits.len()); + for edit in edits { + let chars = self.char_range(&edit.range)?; + resolved.push((edit.range.clone(), chars, edit.text.as_str())); + } + resolved.sort_by_key(|(bytes, _, _)| (bytes.start, bytes.end)); + for pair in resolved.windows(2) { + if let [(first, _, _), (second, _, _)] = pair + && first.end > second.start + { + return Err(TextError::OverlappingEdits { + first: first.clone(), + second: second.clone(), + }); + } + } + let mut rope = self.rope.clone(); + for (_, chars, text) in resolved.iter().rev() { + rope.try_remove(chars.clone())?; + rope.try_insert(chars.start, text)?; + } + self.rope = rope; + Ok(()) + } + + /// Converts a byte offset on a character boundary to a char index. + fn offset_to_char(&self, offset: usize) -> Result { + let len = self.len_bytes(); + if offset > len { + return Err(TextError::OffsetOutOfRange { offset, len }); + } + let char_idx = self.rope.try_byte_to_char(offset)?; + if self.rope.try_char_to_byte(char_idx)? != offset { + return Err(TextError::NotCharBoundary { offset }); + } + Ok(char_idx) + } + + /// Converts a byte range on character boundaries to a char range. + fn char_range(&self, range: &Range) -> Result, TextError> { + if range.start > range.end { + return Err(TextError::InvalidRange { + start: range.start, + end: range.end, + }); + } + let end = self.offset_to_char(range.end)?; + let start = self.offset_to_char(range.start)?; + Ok(start..end) + } +} + +impl From<&str> for Buffer { + fn from(text: &str) -> Self { + Self::new(text) + } +} + +/// Length in bytes of the line break at the end of `line`: 2 for `\r\n`, +/// 1 for `\n` or `\r`, 0 for the last line. +fn line_break_len(line: &RopeSlice<'_>) -> usize { + let len = line.len_chars(); + let last = len.checked_sub(1).and_then(|i| line.get_char(i)); + let before_last = len.checked_sub(2).and_then(|i| line.get_char(i)); + match (before_last, last) { + (Some('\r'), Some('\n')) => 2, + (_, Some('\n' | '\r')) => 1, + _ => 0, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + const UNITS: [ColumnUnit; 2] = [ColumnUnit::Utf16, ColumnUnit::Char]; + + fn pos(line: usize, column: usize) -> Position { + Position::new(line, column) + } + + #[test] + fn counts_bytes_chars_utf16_and_lines() { + let b = Buffer::new("a😀\n漢e\u{301}\r\nz"); + assert_eq!(b.len_bytes(), "a😀\n漢e\u{301}\r\nz".len()); + assert_eq!(b.len_chars(), 9); + assert_eq!(b.len_utf16(), 10); + assert_eq!(b.line_count(), 3); + assert_eq!(b.text(), "a😀\n漢e\u{301}\r\nz"); + assert!(!b.is_empty()); + } + + #[test] + fn empty_text_has_one_empty_line() { + let b = Buffer::new(""); + assert!(b.is_empty()); + assert_eq!(b.line_count(), 1); + assert_eq!(b.line_range(0), Ok(0..0)); + assert_eq!(b.offset_to_position(0, ColumnUnit::Utf16), Ok(pos(0, 0))); + assert_eq!(b.position_to_offset(pos(0, 0), ColumnUnit::Char), Ok(0)); + } + + #[test] + fn trailing_line_break_adds_an_empty_last_line() { + let b = Buffer::new("a\n"); + assert_eq!(b.line_count(), 2); + assert_eq!(b.line_range(1), Ok(2..2)); + assert_eq!(b.offset_to_position(2, ColumnUnit::Char), Ok(pos(1, 0))); + } + + #[test] + fn all_three_line_breaks_split_lines_and_unicode_separators_do_not() { + let b = Buffer::new("a\nb\r\nc\rd\u{2028}e\u{85}f"); + assert_eq!(b.line_count(), 4); + assert_eq!(b.line_text(0).unwrap(), "a"); + assert_eq!(b.line_text(1).unwrap(), "b"); + assert_eq!(b.line_text(2).unwrap(), "c"); + assert_eq!(b.line_text(3).unwrap(), "d\u{2028}e\u{85}f"); + } + + #[test] + fn columns_differ_by_unit_for_astral_chars() { + let b = Buffer::new("x\n😀a"); + let after_a = "x\n😀a".len(); + assert_eq!( + b.offset_to_position(after_a, ColumnUnit::Utf16), + Ok(pos(1, 3)) + ); + assert_eq!( + b.offset_to_position(after_a, ColumnUnit::Char), + Ok(pos(1, 2)) + ); + assert_eq!( + b.position_to_offset(pos(1, 3), ColumnUnit::Utf16), + Ok(after_a) + ); + assert_eq!( + b.position_to_offset(pos(1, 2), ColumnUnit::Char), + Ok(after_a) + ); + } + + #[test] + fn combining_marks_and_cjk_count_one_char_each() { + let b = Buffer::new("e\u{301}漢"); + assert_eq!(b.offset_to_position(3, ColumnUnit::Char), Ok(pos(0, 2))); + assert_eq!(b.offset_to_position(6, ColumnUnit::Utf16), Ok(pos(0, 3))); + } + + #[test] + fn offsets_inside_chars_or_past_the_end_are_errors() { + let b = Buffer::new("😀"); + for unit in UNITS { + assert_eq!( + b.offset_to_position(1, unit), + Err(TextError::NotCharBoundary { offset: 1 }) + ); + assert_eq!( + b.offset_to_position(5, unit), + Err(TextError::OffsetOutOfRange { offset: 5, len: 4 }) + ); + assert!(b.offset_to_position(usize::MAX, unit).is_err()); + } + } + + #[test] + fn offset_inside_crlf_is_an_error() { + let b = Buffer::new("a\r\nb"); + assert_eq!(b.offset_to_position(1, ColumnUnit::Char), Ok(pos(0, 1))); + assert_eq!( + b.offset_to_position(2, ColumnUnit::Char), + Err(TextError::InsideLineBreak { offset: 2 }) + ); + assert_eq!(b.offset_to_position(3, ColumnUnit::Char), Ok(pos(1, 0))); + } + + #[test] + fn bad_positions_are_errors() { + let b = Buffer::new("😀\nab"); + assert_eq!( + b.position_to_offset(pos(2, 0), ColumnUnit::Char), + Err(TextError::LineOutOfRange { + line: 2, + line_count: 2 + }) + ); + assert_eq!( + b.position_to_offset(pos(0, 2), ColumnUnit::Char), + Err(TextError::ColumnOutOfRange { + line: 0, + column: 2, + line_len: 1 + }) + ); + assert_eq!( + b.position_to_offset(pos(0, 1), ColumnUnit::Utf16), + Err(TextError::ColumnInsideChar { line: 0, column: 1 }) + ); + assert_eq!(b.position_to_offset(pos(0, 2), ColumnUnit::Utf16), Ok(4)); + assert!( + b.position_to_offset(pos(usize::MAX, usize::MAX), ColumnUnit::Utf16) + .is_err() + ); + assert!( + b.position_to_offset(pos(1, usize::MAX), ColumnUnit::Char) + .is_err() + ); + } + + #[test] + fn absolute_utf16_offsets() { + let b = Buffer::new("a😀b"); + assert_eq!(b.offset_to_utf16(5), Ok(3)); + assert_eq!(b.utf16_to_offset(3), Ok(5)); + assert_eq!( + b.utf16_to_offset(2), + Err(TextError::Utf16InsideChar { offset: 2 }) + ); + assert_eq!( + b.utf16_to_offset(5), + Err(TextError::Utf16OutOfRange { offset: 5, len: 4 }) + ); + assert_eq!( + b.offset_to_utf16(2), + Err(TextError::NotCharBoundary { offset: 2 }) + ); + } + + #[test] + fn slice_checks_its_range() { + let b = Buffer::new("héllo"); + assert_eq!(b.slice(1..3).unwrap(), "é"); + assert_eq!(b.slice(2..3), Err(TextError::NotCharBoundary { offset: 2 })); + assert_eq!( + b.slice(Range { start: 3, end: 1 }), + Err(TextError::InvalidRange { start: 3, end: 1 }) + ); + assert!(b.slice(0..99).is_err()); + } + + #[test] + fn apply_insert_delete_replace() { + let mut b = Buffer::new("hello world"); + b.apply(&[Edit::replace(0..5, "goodbye")]).unwrap(); + assert_eq!(b.text(), "goodbye world"); + b.apply(&[Edit::insert(13, "!")]).unwrap(); + assert_eq!(b.text(), "goodbye world!"); + b.apply(&[Edit::delete(7..13)]).unwrap(); + assert_eq!(b.text(), "goodbye!"); + b.apply(&[]).unwrap(); + assert_eq!(b.text(), "goodbye!"); + } + + #[test] + fn batch_ranges_refer_to_the_original_text_in_any_order() { + let mut b = Buffer::new("abcdef"); + b.apply(&[ + Edit::replace(4..6, "EF"), + Edit::replace(0..1, "AA"), + Edit::delete(2..3), + ]) + .unwrap(); + assert_eq!(b.text(), "AAbdEF"); + } + + #[test] + fn insertions_at_one_offset_keep_their_order() { + let mut b = Buffer::new("ab"); + b.apply(&[ + Edit::insert(1, "1"), + Edit::replace(1..2, "B"), + Edit::insert(1, "2"), + ]) + .unwrap(); + assert_eq!(b.text(), "a12B"); + } + + #[test] + fn overlapping_edits_are_rejected_and_leave_the_buffer_unchanged() { + let mut b = Buffer::new("abcdef"); + assert_eq!( + b.apply(&[Edit::delete(3..5), Edit::delete(1..4)]), + Err(TextError::OverlappingEdits { + first: 1..4, + second: 3..5 + }) + ); + assert_eq!(b.text(), "abcdef"); + assert!( + b.apply(&[Edit::delete(1..4), Edit::insert(2, "x")]) + .is_err() + ); + assert_eq!(b.text(), "abcdef"); + } + + #[test] + fn invalid_edits_are_rejected_and_leave_the_buffer_unchanged() { + let mut b = Buffer::new("a😀"); + let bad = [ + Edit::insert(2, "x"), + Edit::delete(0..9), + Edit::replace(Range { start: 1, end: 0 }, "x"), + Edit::insert(usize::MAX, "x"), + ]; + for edit in bad { + assert!(b.apply(&[Edit::insert(0, "ok"), edit]).is_err()); + assert_eq!(b.text(), "a😀"); + } + } + + #[test] + fn edits_may_split_and_join_crlf() { + let mut b = Buffer::new("a\r\nb"); + b.apply(&[Edit::insert(2, "x")]).unwrap(); + assert_eq!(b.line_count(), 3); + b.apply(&[Edit::delete(2..3)]).unwrap(); + assert_eq!(b.line_count(), 2); + assert_eq!(b.line_text(1).unwrap(), "b"); + } + + #[test] + fn keeps_the_files_line_ending_on_save() { + let mut crlf = Buffer::new("a\r\nb\r\n"); + assert_eq!(crlf.line_ending(), LineEnding::CrLf); + crlf.apply(&[Edit::insert(1, "\nnew")]).unwrap(); + assert_eq!(crlf.save_text(), "a\r\nnew\r\nb\r\n"); + + let mut cr = Buffer::new("a\rb"); + assert_eq!(cr.line_ending(), LineEnding::Cr); + cr.apply(&[Edit::insert(3, "\r\nc")]).unwrap(); + assert_eq!(cr.save_text(), "a\rb\rc"); + + let plain = Buffer::new("no break"); + assert_eq!(plain.line_ending(), LineEnding::Lf); + + let mut lf = Buffer::new("a\nb"); + lf.set_line_ending(LineEnding::CrLf); + assert_eq!(lf.save_text(), "a\r\nb"); + assert_eq!(lf.text(), "a\nb"); + } +} diff --git a/crates/text/src/edit.rs b/crates/text/src/edit.rs new file mode 100644 index 0000000..ed3936d --- /dev/null +++ b/crates/text/src/edit.rs @@ -0,0 +1,36 @@ +//! Text edits addressed by byte ranges. + +use std::ops::Range; + +/// Replaces the bytes in `range` with `text`. +/// +/// An empty range inserts, an empty `text` deletes. Ranges are byte offsets +/// that must lie on UTF-8 character boundaries of the text the edit is +/// applied to; see [`crate::Buffer::apply`] for how a batch is interpreted. +#[derive(Debug, Clone, PartialEq, Eq, Hash, Default)] +pub struct Edit { + /// Byte range to replace. + pub range: Range, + /// Replacement text. + pub text: String, +} + +impl Edit { + /// Replaces `range` with `text`. + pub fn replace(range: Range, text: impl Into) -> Self { + Self { + range, + text: text.into(), + } + } + + /// Inserts `text` at byte offset `at`. + pub fn insert(at: usize, text: impl Into) -> Self { + Self::replace(at..at, text) + } + + /// Deletes the bytes in `range`. + pub fn delete(range: Range) -> Self { + Self::replace(range, String::new()) + } +} diff --git a/crates/text/src/error.rs b/crates/text/src/error.rs new file mode 100644 index 0000000..83adc1b --- /dev/null +++ b/crates/text/src/error.rs @@ -0,0 +1,151 @@ +//! Errors returned by position conversions and edits. + +use std::fmt; +use std::ops::Range; + +/// Why a position conversion or an edit was rejected. +/// +/// Every conversion and edit on [`crate::Buffer`] returns this instead of +/// panicking, so callers can turn bad offsets from untrusted input (LSP +/// messages, frontend edits, command params) into an error message. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum TextError { + /// A byte offset lies past the end of the text. + OffsetOutOfRange { + /// The rejected byte offset. + offset: usize, + /// The text length in bytes. + len: usize, + }, + /// A byte offset lies inside a multi-byte UTF-8 character. + NotCharBoundary { + /// The rejected byte offset. + offset: usize, + }, + /// A byte offset lies between the `\r` and the `\n` of a CRLF line + /// break, where no line and column exist. + InsideLineBreak { + /// The rejected byte offset. + offset: usize, + }, + /// A UTF-16 offset lies past the end of the text. + Utf16OutOfRange { + /// The rejected offset in UTF-16 code units. + offset: usize, + /// The text length in UTF-16 code units. + len: usize, + }, + /// A UTF-16 offset lies between the two halves of a surrogate pair. + Utf16InsideChar { + /// The rejected offset in UTF-16 code units. + offset: usize, + }, + /// A line index is not smaller than the line count. + LineOutOfRange { + /// The rejected zero-based line index. + line: usize, + /// The number of lines in the text. + line_count: usize, + }, + /// A column lies past the end of its line (line break excluded). + ColumnOutOfRange { + /// The zero-based line index. + line: usize, + /// The rejected column. + column: usize, + /// The line length in the column's unit, line break excluded. + line_len: usize, + }, + /// A UTF-16 column lies between the two halves of a surrogate pair. + ColumnInsideChar { + /// The zero-based line index. + line: usize, + /// The rejected column in UTF-16 code units. + column: usize, + }, + /// A range whose start lies after its end. + InvalidRange { + /// The range start in bytes. + start: usize, + /// The range end in bytes. + end: usize, + }, + /// Two edits of one batch touch overlapping text. + OverlappingEdits { + /// The byte range of the edit that starts first. + first: Range, + /// The byte range of the edit that overlaps it. + second: Range, + }, + /// The rope rejected an operation that validation had accepted; a bug in + /// this crate rather than bad input. + Internal { + /// The rope's own error message. + detail: String, + }, +} + +impl fmt::Display for TextError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::OffsetOutOfRange { offset, len } => { + write!( + f, + "byte offset {offset} is past the end of the text ({len} bytes)" + ) + } + Self::NotCharBoundary { offset } => { + write!(f, "byte offset {offset} is inside a UTF-8 character") + } + Self::InsideLineBreak { offset } => { + write!(f, "byte offset {offset} is inside a CRLF line break") + } + Self::Utf16OutOfRange { offset, len } => write!( + f, + "UTF-16 offset {offset} is past the end of the text ({len} code units)" + ), + Self::Utf16InsideChar { offset } => { + write!(f, "UTF-16 offset {offset} is inside a surrogate pair") + } + Self::LineOutOfRange { line, line_count } => { + write!( + f, + "line {line} does not exist (the text has {line_count} lines)" + ) + } + Self::ColumnOutOfRange { + line, + column, + line_len, + } => write!( + f, + "column {column} is past the end of line {line} (length {line_len})" + ), + Self::ColumnInsideChar { line, column } => { + write!( + f, + "column {column} of line {line} is inside a surrogate pair" + ) + } + Self::InvalidRange { start, end } => { + write!(f, "range start {start} is after its end {end}") + } + Self::OverlappingEdits { first, second } => write!( + f, + "edits {}..{} and {}..{} overlap", + first.start, first.end, second.start, second.end + ), + Self::Internal { detail } => write!(f, "internal text buffer error: {detail}"), + } + } +} + +impl std::error::Error for TextError {} + +impl From for TextError { + fn from(error: ropey::Error) -> Self { + Self::Internal { + detail: error.to_string(), + } + } +} diff --git a/crates/text/src/lib.rs b/crates/text/src/lib.rs index 3e495c7..08dc3c7 100644 --- a/crates/text/src/lib.rs +++ b/crates/text/src/lib.rs @@ -3,3 +3,22 @@ //! Holds the rope buffer, line and column positions, UTF-8/UTF-16 offset //! conversion and edit application. Every other crate converts text offsets //! through this crate instead of slicing strings by hand. +//! +//! - [`Buffer`]: a [`ropey`] rope plus the file's [`LineEnding`]. +//! - [`Position`] and [`ColumnUnit`]: zero-based line and column, the +//! column in UTF-16 code units (LSP, CodeMirror) or in chars. +//! - [`Edit`] and [`Buffer::apply`]: atomic batches of byte-range edits. +//! - [`TextError`]: every conversion and edit is fallible instead of +//! panicking on bad offsets. + +mod buffer; +mod edit; +mod error; +pub mod line_ending; +mod position; + +pub use buffer::Buffer; +pub use edit::Edit; +pub use error::TextError; +pub use line_ending::LineEnding; +pub use position::{ColumnUnit, Position}; diff --git a/crates/text/src/line_ending.rs b/crates/text/src/line_ending.rs new file mode 100644 index 0000000..9a7625e --- /dev/null +++ b/crates/text/src/line_ending.rs @@ -0,0 +1,178 @@ +//! Line ending detection and normalisation. +//! +//! Lutra recognises exactly the three line breaks that LSP and CodeMirror +//! know: `\n`, `\r\n` and a lone `\r`. Unicode separators such as U+2028 are +//! ordinary characters. +//! +//! Save policy: a buffer remembers the line ending its file was opened with +//! ([`LineEnding::detect`], falling back to [`LineEnding::Lf`] for files +//! without any line break) and [`crate::Buffer::save_text`] writes every line +//! break with that ending. Line breaks typed in the editor therefore take the +//! file's ending, and a file keeps its ending across edits. A file with mixed +//! endings is saved with its dominant one. + +/// One of the three recognised line breaks. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Default)] +pub enum LineEnding { + /// `\n`, the Unix ending and the one PSR-12 requires for PHP files. + #[default] + Lf, + /// `\r\n`, the Windows ending. + CrLf, + /// A lone `\r`, the classic Mac OS ending. + Cr, +} + +impl LineEnding { + /// The line break as text. + pub const fn as_str(self) -> &'static str { + match self { + Self::Lf => "\n", + Self::CrLf => "\r\n", + Self::Cr => "\r", + } + } + + /// Returns the most frequent line ending in `text`, or `None` if the + /// text has no line break. Ties go to `Lf`, then `CrLf`, then `Cr`. + pub fn detect(text: &str) -> Option { + Self::detect_chunks(std::iter::once(text)) + } + + /// Like [`LineEnding::detect`] for text split into chunks, such as the + /// chunks of a rope. A CRLF split across two chunks counts once. + pub fn detect_chunks<'a>(chunks: impl IntoIterator) -> Option { + let mut counts = Counts::default(); + let mut pending_cr = false; + for ch in chunks.into_iter().flat_map(str::chars) { + if pending_cr { + pending_cr = false; + if ch == '\n' { + counts.crlf += 1; + continue; + } + counts.cr += 1; + } + match ch { + '\r' => pending_cr = true, + '\n' => counts.lf += 1, + _ => {} + } + } + if pending_cr { + counts.cr += 1; + } + counts.dominant() + } + + /// Returns `text` with every line break replaced by this ending. + pub fn normalize(self, text: &str) -> String { + self.normalize_chunks(std::iter::once(text)) + } + + /// Like [`LineEnding::normalize`] for text split into chunks. A CRLF + /// split across two chunks becomes a single line break. + pub fn normalize_chunks<'a>(self, chunks: impl IntoIterator) -> String { + let ending = self.as_str(); + let mut out = String::new(); + let mut pending_cr = false; + for chunk in chunks { + out.reserve(chunk.len()); + for ch in chunk.chars() { + if pending_cr { + pending_cr = false; + out.push_str(ending); + if ch == '\n' { + continue; + } + } + match ch { + '\r' => pending_cr = true, + '\n' => out.push_str(ending), + other => out.push(other), + } + } + } + if pending_cr { + out.push_str(ending); + } + out + } +} + +/// Number of line breaks of each kind. +#[derive(Default)] +struct Counts { + /// Count of `\n` not preceded by `\r`. + lf: usize, + /// Count of `\r\n`. + crlf: usize, + /// Count of `\r` not followed by `\n`. + cr: usize, +} + +impl Counts { + /// The most frequent ending, ties broken in declaration order. + fn dominant(&self) -> Option { + let mut best: Option<(LineEnding, usize)> = None; + for (ending, count) in [ + (LineEnding::Lf, self.lf), + (LineEnding::CrLf, self.crlf), + (LineEnding::Cr, self.cr), + ] { + if count > best.map_or(0, |(_, n)| n) { + best = Some((ending, count)); + } + } + best.map(|(ending, _)| ending) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn detects_each_ending() { + assert_eq!(LineEnding::detect("a\nb"), Some(LineEnding::Lf)); + assert_eq!(LineEnding::detect("a\r\nb"), Some(LineEnding::CrLf)); + assert_eq!(LineEnding::detect("a\rb"), Some(LineEnding::Cr)); + assert_eq!(LineEnding::detect("a\r"), Some(LineEnding::Cr)); + assert_eq!(LineEnding::detect("no break"), None); + assert_eq!(LineEnding::detect(""), None); + } + + #[test] + fn detect_takes_the_majority_and_breaks_ties_in_order() { + assert_eq!(LineEnding::detect("a\r\nb\r\nc\n"), Some(LineEnding::CrLf)); + assert_eq!(LineEnding::detect("a\r\nb\n"), Some(LineEnding::Lf)); + assert_eq!(LineEnding::detect("a\rb\r\n"), Some(LineEnding::CrLf)); + assert_eq!(LineEnding::detect("\r\r\n\n"), Some(LineEnding::Lf)); + } + + #[test] + fn unicode_separators_are_not_line_breaks() { + assert_eq!(LineEnding::detect("a\u{2028}b\u{85}c"), None); + } + + #[test] + fn crlf_split_across_chunks_counts_once() { + assert_eq!( + LineEnding::detect_chunks(["a\r", "\nb\r", "\n"]), + Some(LineEnding::CrLf) + ); + assert_eq!( + LineEnding::Lf.normalize_chunks(["a\r", "\nb"]), + "a\nb".to_string() + ); + } + + #[test] + fn normalize_rewrites_every_break() { + let mixed = "a\nb\r\nc\rd\r"; + assert_eq!(LineEnding::Lf.normalize(mixed), "a\nb\nc\nd\n"); + assert_eq!(LineEnding::CrLf.normalize(mixed), "a\r\nb\r\nc\r\nd\r\n"); + assert_eq!(LineEnding::Cr.normalize(mixed), "a\rb\rc\rd\r"); + assert_eq!(LineEnding::CrLf.normalize("\r\r\n"), "\r\n\r\n"); + } +} diff --git a/crates/text/src/position.rs b/crates/text/src/position.rs new file mode 100644 index 0000000..268b6b7 --- /dev/null +++ b/crates/text/src/position.rs @@ -0,0 +1,39 @@ +//! Line and column positions. +//! +//! LSP counts columns in UTF-16 code units by default +//! () +//! and CodeMirror addresses text in UTF-16 code units +//! (), so UTF-16 is the unit +//! that crosses the frontend and LSP boundaries. Chars (Unicode scalar +//! values) are offered for Rust-side code that walks text by `char`. + +/// The unit a column is counted in. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum ColumnUnit { + /// UTF-16 code units: characters outside the Basic Multilingual Plane, + /// such as most emoji, count two. + Utf16, + /// Unicode scalar values (Rust `char`s): every code point counts one, + /// including each combining mark. + Char, +} + +/// A zero-based line and column. +/// +/// The column's unit is not stored; conversions take a [`ColumnUnit`]. A +/// column never reaches into the line break: the largest valid column is +/// the line's length without its line break. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Default)] +pub struct Position { + /// Zero-based line index. + pub line: usize, + /// Zero-based column in the unit given to the conversion. + pub column: usize, +} + +impl Position { + /// Creates a position from a zero-based line and column. + pub const fn new(line: usize, column: usize) -> Self { + Self { line, column } + } +} diff --git a/crates/text/tests/proptests.rs b/crates/text/tests/proptests.rs new file mode 100644 index 0000000..995efc2 --- /dev/null +++ b/crates/text/tests/proptests.rs @@ -0,0 +1,241 @@ +//! Property tests: conversions round-trip on random Unicode and random +//! edits, valid or not, never panic and match a plain `String` model. + +#![allow( + clippy::unwrap_used, + clippy::expect_used, + clippy::indexing_slicing, + clippy::string_slice +)] + +use lutra_text::{Buffer, ColumnUnit, Edit, LineEnding, Position, TextError}; +use proptest::prelude::*; + +const UNITS: [ColumnUnit; 2] = [ColumnUnit::Utf16, ColumnUnit::Char]; + +/// Text built from pieces that stress positions: ASCII, all three line +/// breaks, emoji (astral, ZWJ sequences, flags), CJK, combining marks, +/// Unicode separators that are not line breaks, and the highest code point. +fn tricky_text() -> impl Strategy { + let piece = prop_oneof![ + Just("a"), + Just("Z"), + Just(" "), + Just("\t"), + Just("\n"), + Just("\r\n"), + Just("\r"), + Just("😀"), + Just("👨\u{200d}👩\u{200d}👧"), + Just("🇩🇪"), + Just("漢"), + Just("字"), + Just("e\u{301}"), + Just("\u{301}"), + Just("\u{2028}"), + Just("\u{85}"), + Just("\u{10ffff}"), + Just("é"), + ]; + prop::collection::vec(piece, 0..40).prop_map(|pieces| pieces.concat()) +} + +/// Tricky text or fully random strings. +fn any_text() -> impl Strategy { + prop_oneof![3 => tricky_text(), 1 => any::()] +} + +/// Edit ranges anywhere, including past the end and reversed. +fn any_edit(max: usize) -> impl Strategy { + (0..=max + 3, 0..=max + 3, tricky_text()) + .prop_map(|(start, end, text)| Edit::replace(start..end, text)) +} + +/// The line breaks of `text` as byte ranges, CRLF as one break. +fn line_breaks(text: &str) -> Vec> { + let bytes = text.as_bytes(); + let mut breaks = Vec::new(); + let mut i = 0; + while i < bytes.len() { + match bytes[i] { + b'\r' if bytes.get(i + 1) == Some(&b'\n') => { + breaks.push(i..i + 2); + i += 2; + } + b'\r' | b'\n' => { + breaks.push(i..i + 1); + i += 1; + } + _ => i += 1, + } + } + breaks +} + +/// Reference position of a byte offset, computed with `str` methods. +fn model_position(text: &str, offset: usize, unit: ColumnUnit) -> Option { + if !text.is_char_boundary(offset) { + return None; + } + let breaks = line_breaks(text); + if breaks.iter().any(|b| b.start < offset && offset < b.end) { + return None; + } + let line = breaks.iter().filter(|b| b.end <= offset).count(); + let line_start = breaks + .iter() + .filter(|b| b.end <= offset) + .map(|b| b.end) + .next_back() + .unwrap_or(0); + let prefix = &text[line_start..offset]; + let column = match unit { + ColumnUnit::Utf16 => prefix.encode_utf16().count(), + ColumnUnit::Char => prefix.chars().count(), + }; + Some(Position::new(line, column)) +} + +/// Applies edits to a `String` with the documented batch semantics, or +/// `None` where the buffer must reject the batch. +fn model_apply(text: &str, edits: &[Edit]) -> Option { + let mut sorted: Vec<&Edit> = edits.iter().collect(); + for e in &sorted { + if e.range.start > e.range.end + || !text.is_char_boundary(e.range.start) + || !text.is_char_boundary(e.range.end) + { + return None; + } + } + sorted.sort_by_key(|e| (e.range.start, e.range.end)); + if sorted.windows(2).any(|w| w[0].range.end > w[1].range.start) { + return None; + } + let mut out = String::new(); + let mut at = 0; + for e in sorted { + out.push_str(&text[at..e.range.start]); + out.push_str(&e.text); + at = e.range.end; + } + out.push_str(&text[at..]); + Some(out) +} + +proptest! { + #![proptest_config(ProptestConfig::with_cases(512))] + + #[test] + fn counts_match_the_string(text in any_text()) { + let b = Buffer::new(&text); + prop_assert_eq!(b.text(), text.clone()); + prop_assert_eq!(b.len_bytes(), text.len()); + prop_assert_eq!(b.len_chars(), text.chars().count()); + prop_assert_eq!(b.len_utf16(), text.encode_utf16().count()); + prop_assert_eq!(b.line_count(), line_breaks(&text).len() + 1); + } + + #[test] + fn every_byte_offset_converts_like_the_model_and_round_trips(text in any_text()) { + let b = Buffer::new(&text); + for offset in 0..=text.len() + 2 { + for unit in UNITS { + let got = b.offset_to_position(offset, unit); + prop_assert_eq!(got.as_ref().ok().copied(), model_position(&text, offset, unit)); + if let Ok(position) = got { + prop_assert_eq!(b.position_to_offset(position, unit), Ok(offset)); + } + } + let utf16 = b.offset_to_utf16(offset); + prop_assert_eq!(utf16.is_ok(), text.is_char_boundary(offset)); + if let Ok(cu) = utf16 { + prop_assert_eq!(cu, text[..offset].encode_utf16().count()); + prop_assert_eq!(b.utf16_to_offset(cu), Ok(offset)); + } + } + } + + #[test] + fn random_positions_never_panic_and_round_trip( + text in any_text(), + line in 0usize..12, + column in 0usize..48, + ) { + let b = Buffer::new(&text); + for unit in UNITS { + if let Ok(offset) = b.position_to_offset(Position::new(line, column), unit) { + prop_assert_eq!( + b.offset_to_position(offset, unit), + Ok(Position::new(line, column)) + ); + } + } + for cu in 0..=b.len_utf16() + 2 { + if let Ok(offset) = b.utf16_to_offset(cu) { + prop_assert_eq!(b.offset_to_utf16(offset), Ok(cu)); + } + } + } + + #[test] + fn random_edit_batches_never_panic_and_match_the_model( + text in tricky_text(), + edits in prop::collection::vec(any_edit(160), 0..5), + ) { + let mut b = Buffer::new(&text); + let expected = model_apply(&text, &edits); + let result = b.apply(&edits); + prop_assert_eq!(result.is_ok(), expected.is_some(), "{:?}", result); + prop_assert_eq!(b.text(), expected.unwrap_or(text)); + prop_assert_eq!(b.len_bytes(), b.text().len()); + prop_assert_eq!(b.line_count(), line_breaks(&b.text()).len() + 1); + } + + #[test] + fn sequences_of_single_edits_stay_consistent( + text in tricky_text(), + edits in prop::collection::vec(any_edit(80), 0..20), + ) { + let mut b = Buffer::new(&text); + let mut model = text; + for edit in edits { + let expected = model_apply(&model, std::slice::from_ref(&edit)); + let result = b.apply(std::slice::from_ref(&edit)); + prop_assert_eq!(result.is_ok(), expected.is_some()); + if let Some(next) = expected { + model = next; + } + prop_assert_eq!(b.text(), model.clone()); + } + for line in 0..b.line_count() { + let range = b.line_range(line).unwrap(); + for unit in UNITS { + let start = b.offset_to_position(range.start, unit).unwrap(); + prop_assert_eq!(start, Position::new(line, 0)); + let end = b.offset_to_position(range.end, unit).unwrap(); + prop_assert_eq!(b.position_to_offset(end, unit), Ok(range.end)); + let past_end = Position::new(line, end.column + 1); + let is_out_of_range = matches!( + b.position_to_offset(past_end, unit), + Err(TextError::ColumnOutOfRange { .. }) + ); + prop_assert!(is_out_of_range); + } + } + } + + #[test] + fn save_text_uses_only_the_buffers_line_ending(text in tricky_text()) { + let b = Buffer::new(&text); + let ending = b.line_ending(); + prop_assert_eq!(Some(ending), LineEnding::detect(&text).or(Some(LineEnding::Lf))); + let saved = b.save_text(); + let breaks = line_breaks(&saved); + prop_assert_eq!(breaks.len(), line_breaks(&text).len()); + for range in breaks { + prop_assert_eq!(&saved[range], ending.as_str()); + } + prop_assert_eq!(Buffer::new(&saved).text(), saved); + } +} diff --git a/docs/gaps.md b/docs/gaps.md index ac33914..b369a2e 100644 --- a/docs/gaps.md +++ b/docs/gaps.md @@ -1,6 +1,6 @@ # Gaps -> **Status:** M0 closed G1; estimates added · **Last updated:** 2026-10-14 +> **Status:** M0 closed G1; G2 under way (text buffer) · **Last updated:** 2026-10-14 Every known shortfall, ranked. This is the work list: unless the owner gives a task, take the highest open gap. Hours are agent wall-clock hours for one agent, all estimated. @@ -23,6 +23,8 @@ Every known shortfall, ranked. This is the work list: unless the owner gives a t | G15 | Database tools, HTTP client, Docker and remote | later | ~430 h | | G16 | Plug-ins and ecosystem | later | ~120 h | +G2 progress: the engine text buffer landed (M1.1, `crates/text`: rope, UTF-16 and char positions, atomic edit batches, line ending policy). Engine buffers, undo, multi-caret, save and the recovery journal are still open; the estimate stays ~30 h. + Known smaller items from M0: - The CI workflow has never run: the repository has no remote until the owner decides on hosting. @@ -33,5 +35,6 @@ Known smaller items from M0: | Date | Change | Summary | |---|---|---| +| 2026-10-14 | minor | G2 note: text buffer landed (M1.1) | | 2026-10-14 | minor | G1 closed; estimates for G2–G16; M0 follow-ups | | 2026-10-13 | major | Initial gap list | diff --git a/docs/target-app-parity.md b/docs/target-app-parity.md index e83b7dc..0be9e7e 100644 --- a/docs/target-app-parity.md +++ b/docs/target-app-parity.md @@ -1,6 +1,6 @@ # Target-app parity -> **Status:** baseline after M0, no PHP feature ships · **Last updated:** 2026-10-14 · **Target:** PhpStorm 2026.x +> **Status:** M1 in progress (text buffer landed), no PHP feature ships · **Last updated:** 2026-10-14 · **Target:** PhpStorm 2026.x ## How this is measured @@ -12,7 +12,7 @@ Hours are the estimated agent wall-clock hours for one agent to reach behaviour | Area | Weight | Score | Missing | Hours (one agent) | Evidence | |---|---:|---:|---|---:|---| -| Editor core | 12 | 0% | everything | ~120 (est.) | — | +| Editor core | 12 | 0% | everything user-visible; only the engine text buffer exists | ~120 (est.) | `crates/text`: rope buffer, positions, edits (M1.1, unit tests + proptests) | | Project view, VFS | 6 | 0% | everything | ~40 (est.) | — | | Navigation | 10 | 0% | everything | ~120 (est.) | — | | Search and replace | 6 | 0% | everything | ~80 (est.) | — | @@ -48,9 +48,18 @@ No PHP feature. The foundation, verified on Windows 11: - Control channel: `app.ping`, `engine.execute`, `engine.listCommands`, `ui.screenshot`, with proptests and a TCP integration test (`crates/control`), and checked against the running app. - The desktop window opens themed from Rust tokens and shows the product name and version that the frontend gets from `app.about` over IPC. +## What ships (M1, in progress) + +Nothing user-visible yet, so every score stays 0%. + +- M1.1 text buffer (`crates/text`): `Buffer` on a `ropey` rope with byte, char, UTF-16 and line counts; byte offset to line and column in UTF-16 code units or chars and back, all fallible (out of range, inside a character, inside a CRLF, inside a surrogate pair); atomic `Edit` batches in LSP `TextEdit[]` semantics that reject reversed, out-of-range and overlapping ranges; ` +` / ` +` / ` ` detection with a keep-the-file's-ending save policy. 23 unit tests and 6 proptests (round-trips on emoji, ZWJ sequences, CJK and combining marks; random edit batches checked against a `String` model, never panicking). + ## Revision history | Date | Change | Summary | |---|---|---| +| 2026-10-14 | minor | M1.1 text buffer evidence (Editor core, score unchanged) | | 2026-10-14 | minor | Hours per area, weights reviewed, M0 evidence | | 2026-10-13 | major | Baseline |