M1.1: text buffer with UTF-16 positions, edit batches and line endings

lutra-text gets Buffer on a ropey rope: byte/char/UTF-16/line counts,
byte offset <-> (line, column) in UTF-16 code units or chars, absolute
UTF-16 offsets, atomic Edit batches in LSP TextEdit[] semantics, and
\n / \r\n / \r detection with a keep-the-file's-ending save policy.
All conversions return TextError instead of panicking.

ropey 1.6.1 (MIT) without unicode_lines, so only the LSP/CodeMirror
line breaks split lines. proptest moves into workspace deps.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
danny 2026-10-14 19:20:18 +02:00
parent faadb05eb3
commit 5dac918b14
12 changed files with 1278 additions and 3 deletions

View file

@ -12,6 +12,10 @@ publish.workspace = true
layer = 0
[dependencies]
ropey.workspace = true
[dev-dependencies]
proptest.workspace = true
[lints]
workspace = true

573
crates/text/src/buffer.rs Normal file
View file

@ -0,0 +1,573 @@
//! The rope-backed text buffer.
use std::ops::Range;
use ropey::{Rope, RopeSlice};
use crate::{ColumnUnit, Edit, LineEnding, Position, TextError};
/// An editable text held in a rope, with the line ending of its file.
///
/// Offsets are byte offsets into the UTF-8 text. Every conversion checks
/// its input and returns a [`TextError`] for offsets past the end, inside a
/// character or inside a CRLF line break, so offsets from untrusted sources
/// can be passed straight in. Line breaks are `\n`, `\r\n` and a lone `\r`
/// (see [`crate::line_ending`]); the line count is the number of line
/// breaks plus one, so a text ending in a line break has an empty last line.
#[derive(Debug, Clone, Default)]
pub struct Buffer {
/// The text.
rope: Rope,
/// The line ending used when saving.
line_ending: LineEnding,
}
impl Buffer {
/// Builds a buffer from `text`, taking the line ending detected in it or
/// [`LineEnding::Lf`] if the text has no line break.
pub fn new(text: &str) -> Self {
Self {
rope: Rope::from_str(text),
line_ending: LineEnding::detect(text).unwrap_or_default(),
}
}
/// The whole text as a string.
pub fn text(&self) -> String {
self.rope.to_string()
}
/// The text in the byte `range`.
///
/// # Errors
/// Fails when the range is reversed, reaches past the end or does not
/// lie on character boundaries.
pub fn slice(&self, range: Range<usize>) -> Result<String, TextError> {
let chars = self.char_range(&range)?;
Ok(self
.rope
.get_slice(chars)
.map(String::from)
.unwrap_or_default())
}
/// Length of the text in bytes.
pub fn len_bytes(&self) -> usize {
self.rope.len_bytes()
}
/// Length of the text in chars (Unicode scalar values).
pub fn len_chars(&self) -> usize {
self.rope.len_chars()
}
/// Length of the text in UTF-16 code units.
pub fn len_utf16(&self) -> usize {
self.rope.len_utf16_cu()
}
/// Number of lines: the number of line breaks plus one.
pub fn line_count(&self) -> usize {
self.rope.len_lines()
}
/// Whether the text is empty.
pub fn is_empty(&self) -> bool {
self.rope.len_bytes() == 0
}
/// The line ending the text is saved with.
pub fn line_ending(&self) -> LineEnding {
self.line_ending
}
/// Changes the line ending the text is saved with.
pub fn set_line_ending(&mut self, line_ending: LineEnding) {
self.line_ending = line_ending;
}
/// The text to write to disk: every line break replaced by the buffer's
/// line ending, so the file keeps its ending.
pub fn save_text(&self) -> String {
self.line_ending.normalize_chunks(self.rope.chunks())
}
/// The byte range of `line` without its line break.
///
/// # Errors
/// [`TextError::LineOutOfRange`] when the line does not exist.
pub fn line_range(&self, line: usize) -> Result<Range<usize>, TextError> {
let slice = self.rope.get_line(line).ok_or(TextError::LineOutOfRange {
line,
line_count: self.line_count(),
})?;
let start = self.rope.try_line_to_byte(line)?;
let content = slice.len_bytes().saturating_sub(line_break_len(&slice));
Ok(start..start.saturating_add(content))
}
/// The text of `line` without its line break.
///
/// # Errors
/// [`TextError::LineOutOfRange`] when the line does not exist.
pub fn line_text(&self, line: usize) -> Result<String, TextError> {
let range = self.line_range(line)?;
self.slice(range)
}
/// Converts a byte offset to a line and a column counted in `unit`.
///
/// # Errors
/// Fails when the offset is past the end, inside a character or between
/// the `\r` and `\n` of a CRLF line break.
pub fn offset_to_position(
&self,
offset: usize,
unit: ColumnUnit,
) -> Result<Position, TextError> {
let char_idx = self.offset_to_char(offset)?;
let line = self.rope.try_char_to_line(char_idx)?;
let range = self.line_range(line)?;
if offset > range.end {
return Err(TextError::InsideLineBreak { offset });
}
let line_start = self.rope.try_byte_to_char(range.start)?;
let column = match unit {
ColumnUnit::Char => char_idx.saturating_sub(line_start),
ColumnUnit::Utf16 => self
.rope
.try_char_to_utf16_cu(char_idx)?
.saturating_sub(self.rope.try_char_to_utf16_cu(line_start)?),
};
Ok(Position { line, column })
}
/// Converts a line and a column counted in `unit` to a byte offset.
///
/// # Errors
/// Fails when the line does not exist, the column is past the end of
/// the line (line break excluded) or, in UTF-16, falls inside a
/// surrogate pair.
pub fn position_to_offset(
&self,
position: Position,
unit: ColumnUnit,
) -> Result<usize, TextError> {
let Position { line, column } = position;
let range = self.line_range(line)?;
let start_char = self.rope.try_byte_to_char(range.start)?;
let end_char = self.rope.try_byte_to_char(range.end)?;
let char_idx = match unit {
ColumnUnit::Char => {
let line_len = end_char.saturating_sub(start_char);
if column > line_len {
return Err(TextError::ColumnOutOfRange {
line,
column,
line_len,
});
}
start_char.saturating_add(column)
}
ColumnUnit::Utf16 => {
let start_cu = self.rope.try_char_to_utf16_cu(start_char)?;
let end_cu = self.rope.try_char_to_utf16_cu(end_char)?;
let line_len = end_cu.saturating_sub(start_cu);
if column > line_len {
return Err(TextError::ColumnOutOfRange {
line,
column,
line_len,
});
}
let target = start_cu.saturating_add(column);
let char_idx = self.rope.try_utf16_cu_to_char(target)?;
if self.rope.try_char_to_utf16_cu(char_idx)? != target {
return Err(TextError::ColumnInsideChar { line, column });
}
char_idx
}
};
Ok(self.rope.try_char_to_byte(char_idx)?)
}
/// Converts a byte offset to an offset in UTF-16 code units from the
/// start of the text. Line breaks count as stored (`\r\n` is two).
///
/// # Errors
/// Fails when the offset is past the end or inside a character.
pub fn offset_to_utf16(&self, offset: usize) -> Result<usize, TextError> {
let char_idx = self.offset_to_char(offset)?;
Ok(self.rope.try_char_to_utf16_cu(char_idx)?)
}
/// Converts an offset in UTF-16 code units from the start of the text to
/// a byte offset.
///
/// # Errors
/// Fails when the offset is past the end or inside a surrogate pair.
pub fn utf16_to_offset(&self, offset: usize) -> Result<usize, TextError> {
let len = self.len_utf16();
if offset > len {
return Err(TextError::Utf16OutOfRange { offset, len });
}
let char_idx = self.rope.try_utf16_cu_to_char(offset)?;
if self.rope.try_char_to_utf16_cu(char_idx)? != offset {
return Err(TextError::Utf16InsideChar { offset });
}
Ok(self.rope.try_char_to_byte(char_idx)?)
}
/// Applies a batch of edits atomically.
///
/// All ranges refer to the text before the call, as in an LSP
/// `TextEdit[]`, and may come in any order. Insertions at the same
/// offset land in the order given, and an insertion at the start of a
/// replaced range lands before the replacement.
///
/// # Errors
/// Rejects the whole batch, leaving the buffer unchanged, when a range
/// is reversed, reaches past the end, does not lie on character
/// boundaries, or overlaps another edit of the batch.
pub fn apply(&mut self, edits: &[Edit]) -> Result<(), TextError> {
let mut resolved = Vec::with_capacity(edits.len());
for edit in edits {
let chars = self.char_range(&edit.range)?;
resolved.push((edit.range.clone(), chars, edit.text.as_str()));
}
resolved.sort_by_key(|(bytes, _, _)| (bytes.start, bytes.end));
for pair in resolved.windows(2) {
if let [(first, _, _), (second, _, _)] = pair
&& first.end > second.start
{
return Err(TextError::OverlappingEdits {
first: first.clone(),
second: second.clone(),
});
}
}
let mut rope = self.rope.clone();
for (_, chars, text) in resolved.iter().rev() {
rope.try_remove(chars.clone())?;
rope.try_insert(chars.start, text)?;
}
self.rope = rope;
Ok(())
}
/// Converts a byte offset on a character boundary to a char index.
fn offset_to_char(&self, offset: usize) -> Result<usize, TextError> {
let len = self.len_bytes();
if offset > len {
return Err(TextError::OffsetOutOfRange { offset, len });
}
let char_idx = self.rope.try_byte_to_char(offset)?;
if self.rope.try_char_to_byte(char_idx)? != offset {
return Err(TextError::NotCharBoundary { offset });
}
Ok(char_idx)
}
/// Converts a byte range on character boundaries to a char range.
fn char_range(&self, range: &Range<usize>) -> Result<Range<usize>, TextError> {
if range.start > range.end {
return Err(TextError::InvalidRange {
start: range.start,
end: range.end,
});
}
let end = self.offset_to_char(range.end)?;
let start = self.offset_to_char(range.start)?;
Ok(start..end)
}
}
impl From<&str> for Buffer {
fn from(text: &str) -> Self {
Self::new(text)
}
}
/// Length in bytes of the line break at the end of `line`: 2 for `\r\n`,
/// 1 for `\n` or `\r`, 0 for the last line.
fn line_break_len(line: &RopeSlice<'_>) -> usize {
let len = line.len_chars();
let last = len.checked_sub(1).and_then(|i| line.get_char(i));
let before_last = len.checked_sub(2).and_then(|i| line.get_char(i));
match (before_last, last) {
(Some('\r'), Some('\n')) => 2,
(_, Some('\n' | '\r')) => 1,
_ => 0,
}
}
#[cfg(test)]
mod tests {
use super::*;
const UNITS: [ColumnUnit; 2] = [ColumnUnit::Utf16, ColumnUnit::Char];
fn pos(line: usize, column: usize) -> Position {
Position::new(line, column)
}
#[test]
fn counts_bytes_chars_utf16_and_lines() {
let b = Buffer::new("a😀\n漢e\u{301}\r\nz");
assert_eq!(b.len_bytes(), "a😀\n漢e\u{301}\r\nz".len());
assert_eq!(b.len_chars(), 9);
assert_eq!(b.len_utf16(), 10);
assert_eq!(b.line_count(), 3);
assert_eq!(b.text(), "a😀\n漢e\u{301}\r\nz");
assert!(!b.is_empty());
}
#[test]
fn empty_text_has_one_empty_line() {
let b = Buffer::new("");
assert!(b.is_empty());
assert_eq!(b.line_count(), 1);
assert_eq!(b.line_range(0), Ok(0..0));
assert_eq!(b.offset_to_position(0, ColumnUnit::Utf16), Ok(pos(0, 0)));
assert_eq!(b.position_to_offset(pos(0, 0), ColumnUnit::Char), Ok(0));
}
#[test]
fn trailing_line_break_adds_an_empty_last_line() {
let b = Buffer::new("a\n");
assert_eq!(b.line_count(), 2);
assert_eq!(b.line_range(1), Ok(2..2));
assert_eq!(b.offset_to_position(2, ColumnUnit::Char), Ok(pos(1, 0)));
}
#[test]
fn all_three_line_breaks_split_lines_and_unicode_separators_do_not() {
let b = Buffer::new("a\nb\r\nc\rd\u{2028}e\u{85}f");
assert_eq!(b.line_count(), 4);
assert_eq!(b.line_text(0).unwrap(), "a");
assert_eq!(b.line_text(1).unwrap(), "b");
assert_eq!(b.line_text(2).unwrap(), "c");
assert_eq!(b.line_text(3).unwrap(), "d\u{2028}e\u{85}f");
}
#[test]
fn columns_differ_by_unit_for_astral_chars() {
let b = Buffer::new("x\n😀a");
let after_a = "x\n😀a".len();
assert_eq!(
b.offset_to_position(after_a, ColumnUnit::Utf16),
Ok(pos(1, 3))
);
assert_eq!(
b.offset_to_position(after_a, ColumnUnit::Char),
Ok(pos(1, 2))
);
assert_eq!(
b.position_to_offset(pos(1, 3), ColumnUnit::Utf16),
Ok(after_a)
);
assert_eq!(
b.position_to_offset(pos(1, 2), ColumnUnit::Char),
Ok(after_a)
);
}
#[test]
fn combining_marks_and_cjk_count_one_char_each() {
let b = Buffer::new("e\u{301}漢");
assert_eq!(b.offset_to_position(3, ColumnUnit::Char), Ok(pos(0, 2)));
assert_eq!(b.offset_to_position(6, ColumnUnit::Utf16), Ok(pos(0, 3)));
}
#[test]
fn offsets_inside_chars_or_past_the_end_are_errors() {
let b = Buffer::new("😀");
for unit in UNITS {
assert_eq!(
b.offset_to_position(1, unit),
Err(TextError::NotCharBoundary { offset: 1 })
);
assert_eq!(
b.offset_to_position(5, unit),
Err(TextError::OffsetOutOfRange { offset: 5, len: 4 })
);
assert!(b.offset_to_position(usize::MAX, unit).is_err());
}
}
#[test]
fn offset_inside_crlf_is_an_error() {
let b = Buffer::new("a\r\nb");
assert_eq!(b.offset_to_position(1, ColumnUnit::Char), Ok(pos(0, 1)));
assert_eq!(
b.offset_to_position(2, ColumnUnit::Char),
Err(TextError::InsideLineBreak { offset: 2 })
);
assert_eq!(b.offset_to_position(3, ColumnUnit::Char), Ok(pos(1, 0)));
}
#[test]
fn bad_positions_are_errors() {
let b = Buffer::new("😀\nab");
assert_eq!(
b.position_to_offset(pos(2, 0), ColumnUnit::Char),
Err(TextError::LineOutOfRange {
line: 2,
line_count: 2
})
);
assert_eq!(
b.position_to_offset(pos(0, 2), ColumnUnit::Char),
Err(TextError::ColumnOutOfRange {
line: 0,
column: 2,
line_len: 1
})
);
assert_eq!(
b.position_to_offset(pos(0, 1), ColumnUnit::Utf16),
Err(TextError::ColumnInsideChar { line: 0, column: 1 })
);
assert_eq!(b.position_to_offset(pos(0, 2), ColumnUnit::Utf16), Ok(4));
assert!(
b.position_to_offset(pos(usize::MAX, usize::MAX), ColumnUnit::Utf16)
.is_err()
);
assert!(
b.position_to_offset(pos(1, usize::MAX), ColumnUnit::Char)
.is_err()
);
}
#[test]
fn absolute_utf16_offsets() {
let b = Buffer::new("a😀b");
assert_eq!(b.offset_to_utf16(5), Ok(3));
assert_eq!(b.utf16_to_offset(3), Ok(5));
assert_eq!(
b.utf16_to_offset(2),
Err(TextError::Utf16InsideChar { offset: 2 })
);
assert_eq!(
b.utf16_to_offset(5),
Err(TextError::Utf16OutOfRange { offset: 5, len: 4 })
);
assert_eq!(
b.offset_to_utf16(2),
Err(TextError::NotCharBoundary { offset: 2 })
);
}
#[test]
fn slice_checks_its_range() {
let b = Buffer::new("héllo");
assert_eq!(b.slice(1..3).unwrap(), "é");
assert_eq!(b.slice(2..3), Err(TextError::NotCharBoundary { offset: 2 }));
assert_eq!(
b.slice(Range { start: 3, end: 1 }),
Err(TextError::InvalidRange { start: 3, end: 1 })
);
assert!(b.slice(0..99).is_err());
}
#[test]
fn apply_insert_delete_replace() {
let mut b = Buffer::new("hello world");
b.apply(&[Edit::replace(0..5, "goodbye")]).unwrap();
assert_eq!(b.text(), "goodbye world");
b.apply(&[Edit::insert(13, "!")]).unwrap();
assert_eq!(b.text(), "goodbye world!");
b.apply(&[Edit::delete(7..13)]).unwrap();
assert_eq!(b.text(), "goodbye!");
b.apply(&[]).unwrap();
assert_eq!(b.text(), "goodbye!");
}
#[test]
fn batch_ranges_refer_to_the_original_text_in_any_order() {
let mut b = Buffer::new("abcdef");
b.apply(&[
Edit::replace(4..6, "EF"),
Edit::replace(0..1, "AA"),
Edit::delete(2..3),
])
.unwrap();
assert_eq!(b.text(), "AAbdEF");
}
#[test]
fn insertions_at_one_offset_keep_their_order() {
let mut b = Buffer::new("ab");
b.apply(&[
Edit::insert(1, "1"),
Edit::replace(1..2, "B"),
Edit::insert(1, "2"),
])
.unwrap();
assert_eq!(b.text(), "a12B");
}
#[test]
fn overlapping_edits_are_rejected_and_leave_the_buffer_unchanged() {
let mut b = Buffer::new("abcdef");
assert_eq!(
b.apply(&[Edit::delete(3..5), Edit::delete(1..4)]),
Err(TextError::OverlappingEdits {
first: 1..4,
second: 3..5
})
);
assert_eq!(b.text(), "abcdef");
assert!(
b.apply(&[Edit::delete(1..4), Edit::insert(2, "x")])
.is_err()
);
assert_eq!(b.text(), "abcdef");
}
#[test]
fn invalid_edits_are_rejected_and_leave_the_buffer_unchanged() {
let mut b = Buffer::new("a😀");
let bad = [
Edit::insert(2, "x"),
Edit::delete(0..9),
Edit::replace(Range { start: 1, end: 0 }, "x"),
Edit::insert(usize::MAX, "x"),
];
for edit in bad {
assert!(b.apply(&[Edit::insert(0, "ok"), edit]).is_err());
assert_eq!(b.text(), "a😀");
}
}
#[test]
fn edits_may_split_and_join_crlf() {
let mut b = Buffer::new("a\r\nb");
b.apply(&[Edit::insert(2, "x")]).unwrap();
assert_eq!(b.line_count(), 3);
b.apply(&[Edit::delete(2..3)]).unwrap();
assert_eq!(b.line_count(), 2);
assert_eq!(b.line_text(1).unwrap(), "b");
}
#[test]
fn keeps_the_files_line_ending_on_save() {
let mut crlf = Buffer::new("a\r\nb\r\n");
assert_eq!(crlf.line_ending(), LineEnding::CrLf);
crlf.apply(&[Edit::insert(1, "\nnew")]).unwrap();
assert_eq!(crlf.save_text(), "a\r\nnew\r\nb\r\n");
let mut cr = Buffer::new("a\rb");
assert_eq!(cr.line_ending(), LineEnding::Cr);
cr.apply(&[Edit::insert(3, "\r\nc")]).unwrap();
assert_eq!(cr.save_text(), "a\rb\rc");
let plain = Buffer::new("no break");
assert_eq!(plain.line_ending(), LineEnding::Lf);
let mut lf = Buffer::new("a\nb");
lf.set_line_ending(LineEnding::CrLf);
assert_eq!(lf.save_text(), "a\r\nb");
assert_eq!(lf.text(), "a\nb");
}
}

36
crates/text/src/edit.rs Normal file
View file

@ -0,0 +1,36 @@
//! Text edits addressed by byte ranges.
use std::ops::Range;
/// Replaces the bytes in `range` with `text`.
///
/// An empty range inserts, an empty `text` deletes. Ranges are byte offsets
/// that must lie on UTF-8 character boundaries of the text the edit is
/// applied to; see [`crate::Buffer::apply`] for how a batch is interpreted.
#[derive(Debug, Clone, PartialEq, Eq, Hash, Default)]
pub struct Edit {
/// Byte range to replace.
pub range: Range<usize>,
/// Replacement text.
pub text: String,
}
impl Edit {
/// Replaces `range` with `text`.
pub fn replace(range: Range<usize>, text: impl Into<String>) -> Self {
Self {
range,
text: text.into(),
}
}
/// Inserts `text` at byte offset `at`.
pub fn insert(at: usize, text: impl Into<String>) -> Self {
Self::replace(at..at, text)
}
/// Deletes the bytes in `range`.
pub fn delete(range: Range<usize>) -> Self {
Self::replace(range, String::new())
}
}

151
crates/text/src/error.rs Normal file
View file

@ -0,0 +1,151 @@
//! Errors returned by position conversions and edits.
use std::fmt;
use std::ops::Range;
/// Why a position conversion or an edit was rejected.
///
/// Every conversion and edit on [`crate::Buffer`] returns this instead of
/// panicking, so callers can turn bad offsets from untrusted input (LSP
/// messages, frontend edits, command params) into an error message.
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum TextError {
/// A byte offset lies past the end of the text.
OffsetOutOfRange {
/// The rejected byte offset.
offset: usize,
/// The text length in bytes.
len: usize,
},
/// A byte offset lies inside a multi-byte UTF-8 character.
NotCharBoundary {
/// The rejected byte offset.
offset: usize,
},
/// A byte offset lies between the `\r` and the `\n` of a CRLF line
/// break, where no line and column exist.
InsideLineBreak {
/// The rejected byte offset.
offset: usize,
},
/// A UTF-16 offset lies past the end of the text.
Utf16OutOfRange {
/// The rejected offset in UTF-16 code units.
offset: usize,
/// The text length in UTF-16 code units.
len: usize,
},
/// A UTF-16 offset lies between the two halves of a surrogate pair.
Utf16InsideChar {
/// The rejected offset in UTF-16 code units.
offset: usize,
},
/// A line index is not smaller than the line count.
LineOutOfRange {
/// The rejected zero-based line index.
line: usize,
/// The number of lines in the text.
line_count: usize,
},
/// A column lies past the end of its line (line break excluded).
ColumnOutOfRange {
/// The zero-based line index.
line: usize,
/// The rejected column.
column: usize,
/// The line length in the column's unit, line break excluded.
line_len: usize,
},
/// A UTF-16 column lies between the two halves of a surrogate pair.
ColumnInsideChar {
/// The zero-based line index.
line: usize,
/// The rejected column in UTF-16 code units.
column: usize,
},
/// A range whose start lies after its end.
InvalidRange {
/// The range start in bytes.
start: usize,
/// The range end in bytes.
end: usize,
},
/// Two edits of one batch touch overlapping text.
OverlappingEdits {
/// The byte range of the edit that starts first.
first: Range<usize>,
/// The byte range of the edit that overlaps it.
second: Range<usize>,
},
/// The rope rejected an operation that validation had accepted; a bug in
/// this crate rather than bad input.
Internal {
/// The rope's own error message.
detail: String,
},
}
impl fmt::Display for TextError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::OffsetOutOfRange { offset, len } => {
write!(
f,
"byte offset {offset} is past the end of the text ({len} bytes)"
)
}
Self::NotCharBoundary { offset } => {
write!(f, "byte offset {offset} is inside a UTF-8 character")
}
Self::InsideLineBreak { offset } => {
write!(f, "byte offset {offset} is inside a CRLF line break")
}
Self::Utf16OutOfRange { offset, len } => write!(
f,
"UTF-16 offset {offset} is past the end of the text ({len} code units)"
),
Self::Utf16InsideChar { offset } => {
write!(f, "UTF-16 offset {offset} is inside a surrogate pair")
}
Self::LineOutOfRange { line, line_count } => {
write!(
f,
"line {line} does not exist (the text has {line_count} lines)"
)
}
Self::ColumnOutOfRange {
line,
column,
line_len,
} => write!(
f,
"column {column} is past the end of line {line} (length {line_len})"
),
Self::ColumnInsideChar { line, column } => {
write!(
f,
"column {column} of line {line} is inside a surrogate pair"
)
}
Self::InvalidRange { start, end } => {
write!(f, "range start {start} is after its end {end}")
}
Self::OverlappingEdits { first, second } => write!(
f,
"edits {}..{} and {}..{} overlap",
first.start, first.end, second.start, second.end
),
Self::Internal { detail } => write!(f, "internal text buffer error: {detail}"),
}
}
}
impl std::error::Error for TextError {}
impl From<ropey::Error> for TextError {
fn from(error: ropey::Error) -> Self {
Self::Internal {
detail: error.to_string(),
}
}
}

View file

@ -3,3 +3,22 @@
//! Holds the rope buffer, line and column positions, UTF-8/UTF-16 offset
//! conversion and edit application. Every other crate converts text offsets
//! through this crate instead of slicing strings by hand.
//!
//! - [`Buffer`]: a [`ropey`] rope plus the file's [`LineEnding`].
//! - [`Position`] and [`ColumnUnit`]: zero-based line and column, the
//! column in UTF-16 code units (LSP, CodeMirror) or in chars.
//! - [`Edit`] and [`Buffer::apply`]: atomic batches of byte-range edits.
//! - [`TextError`]: every conversion and edit is fallible instead of
//! panicking on bad offsets.
mod buffer;
mod edit;
mod error;
pub mod line_ending;
mod position;
pub use buffer::Buffer;
pub use edit::Edit;
pub use error::TextError;
pub use line_ending::LineEnding;
pub use position::{ColumnUnit, Position};

View file

@ -0,0 +1,178 @@
//! Line ending detection and normalisation.
//!
//! Lutra recognises exactly the three line breaks that LSP and CodeMirror
//! know: `\n`, `\r\n` and a lone `\r`. Unicode separators such as U+2028 are
//! ordinary characters.
//!
//! Save policy: a buffer remembers the line ending its file was opened with
//! ([`LineEnding::detect`], falling back to [`LineEnding::Lf`] for files
//! without any line break) and [`crate::Buffer::save_text`] writes every line
//! break with that ending. Line breaks typed in the editor therefore take the
//! file's ending, and a file keeps its ending across edits. A file with mixed
//! endings is saved with its dominant one.
/// One of the three recognised line breaks.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Default)]
pub enum LineEnding {
/// `\n`, the Unix ending and the one PSR-12 requires for PHP files.
#[default]
Lf,
/// `\r\n`, the Windows ending.
CrLf,
/// A lone `\r`, the classic Mac OS ending.
Cr,
}
impl LineEnding {
/// The line break as text.
pub const fn as_str(self) -> &'static str {
match self {
Self::Lf => "\n",
Self::CrLf => "\r\n",
Self::Cr => "\r",
}
}
/// Returns the most frequent line ending in `text`, or `None` if the
/// text has no line break. Ties go to `Lf`, then `CrLf`, then `Cr`.
pub fn detect(text: &str) -> Option<Self> {
Self::detect_chunks(std::iter::once(text))
}
/// Like [`LineEnding::detect`] for text split into chunks, such as the
/// chunks of a rope. A CRLF split across two chunks counts once.
pub fn detect_chunks<'a>(chunks: impl IntoIterator<Item = &'a str>) -> Option<Self> {
let mut counts = Counts::default();
let mut pending_cr = false;
for ch in chunks.into_iter().flat_map(str::chars) {
if pending_cr {
pending_cr = false;
if ch == '\n' {
counts.crlf += 1;
continue;
}
counts.cr += 1;
}
match ch {
'\r' => pending_cr = true,
'\n' => counts.lf += 1,
_ => {}
}
}
if pending_cr {
counts.cr += 1;
}
counts.dominant()
}
/// Returns `text` with every line break replaced by this ending.
pub fn normalize(self, text: &str) -> String {
self.normalize_chunks(std::iter::once(text))
}
/// Like [`LineEnding::normalize`] for text split into chunks. A CRLF
/// split across two chunks becomes a single line break.
pub fn normalize_chunks<'a>(self, chunks: impl IntoIterator<Item = &'a str>) -> String {
let ending = self.as_str();
let mut out = String::new();
let mut pending_cr = false;
for chunk in chunks {
out.reserve(chunk.len());
for ch in chunk.chars() {
if pending_cr {
pending_cr = false;
out.push_str(ending);
if ch == '\n' {
continue;
}
}
match ch {
'\r' => pending_cr = true,
'\n' => out.push_str(ending),
other => out.push(other),
}
}
}
if pending_cr {
out.push_str(ending);
}
out
}
}
/// Number of line breaks of each kind.
#[derive(Default)]
struct Counts {
/// Count of `\n` not preceded by `\r`.
lf: usize,
/// Count of `\r\n`.
crlf: usize,
/// Count of `\r` not followed by `\n`.
cr: usize,
}
impl Counts {
/// The most frequent ending, ties broken in declaration order.
fn dominant(&self) -> Option<LineEnding> {
let mut best: Option<(LineEnding, usize)> = None;
for (ending, count) in [
(LineEnding::Lf, self.lf),
(LineEnding::CrLf, self.crlf),
(LineEnding::Cr, self.cr),
] {
if count > best.map_or(0, |(_, n)| n) {
best = Some((ending, count));
}
}
best.map(|(ending, _)| ending)
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn detects_each_ending() {
assert_eq!(LineEnding::detect("a\nb"), Some(LineEnding::Lf));
assert_eq!(LineEnding::detect("a\r\nb"), Some(LineEnding::CrLf));
assert_eq!(LineEnding::detect("a\rb"), Some(LineEnding::Cr));
assert_eq!(LineEnding::detect("a\r"), Some(LineEnding::Cr));
assert_eq!(LineEnding::detect("no break"), None);
assert_eq!(LineEnding::detect(""), None);
}
#[test]
fn detect_takes_the_majority_and_breaks_ties_in_order() {
assert_eq!(LineEnding::detect("a\r\nb\r\nc\n"), Some(LineEnding::CrLf));
assert_eq!(LineEnding::detect("a\r\nb\n"), Some(LineEnding::Lf));
assert_eq!(LineEnding::detect("a\rb\r\n"), Some(LineEnding::CrLf));
assert_eq!(LineEnding::detect("\r\r\n\n"), Some(LineEnding::Lf));
}
#[test]
fn unicode_separators_are_not_line_breaks() {
assert_eq!(LineEnding::detect("a\u{2028}b\u{85}c"), None);
}
#[test]
fn crlf_split_across_chunks_counts_once() {
assert_eq!(
LineEnding::detect_chunks(["a\r", "\nb\r", "\n"]),
Some(LineEnding::CrLf)
);
assert_eq!(
LineEnding::Lf.normalize_chunks(["a\r", "\nb"]),
"a\nb".to_string()
);
}
#[test]
fn normalize_rewrites_every_break() {
let mixed = "a\nb\r\nc\rd\r";
assert_eq!(LineEnding::Lf.normalize(mixed), "a\nb\nc\nd\n");
assert_eq!(LineEnding::CrLf.normalize(mixed), "a\r\nb\r\nc\r\nd\r\n");
assert_eq!(LineEnding::Cr.normalize(mixed), "a\rb\rc\rd\r");
assert_eq!(LineEnding::CrLf.normalize("\r\r\n"), "\r\n\r\n");
}
}

View file

@ -0,0 +1,39 @@
//! Line and column positions.
//!
//! LSP counts columns in UTF-16 code units by default
//! (<https://microsoft.github.io/language-server-protocol/specifications/lsp/3.17/specification/#textDocuments>)
//! and CodeMirror addresses text in UTF-16 code units
//! (<https://codemirror.net/docs/ref/#state.Text>), so UTF-16 is the unit
//! that crosses the frontend and LSP boundaries. Chars (Unicode scalar
//! values) are offered for Rust-side code that walks text by `char`.
/// The unit a column is counted in.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub enum ColumnUnit {
/// UTF-16 code units: characters outside the Basic Multilingual Plane,
/// such as most emoji, count two.
Utf16,
/// Unicode scalar values (Rust `char`s): every code point counts one,
/// including each combining mark.
Char,
}
/// A zero-based line and column.
///
/// The column's unit is not stored; conversions take a [`ColumnUnit`]. A
/// column never reaches into the line break: the largest valid column is
/// the line's length without its line break.
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Default)]
pub struct Position {
/// Zero-based line index.
pub line: usize,
/// Zero-based column in the unit given to the conversion.
pub column: usize,
}
impl Position {
/// Creates a position from a zero-based line and column.
pub const fn new(line: usize, column: usize) -> Self {
Self { line, column }
}
}

View file

@ -0,0 +1,241 @@
//! Property tests: conversions round-trip on random Unicode and random
//! edits, valid or not, never panic and match a plain `String` model.
#![allow(
clippy::unwrap_used,
clippy::expect_used,
clippy::indexing_slicing,
clippy::string_slice
)]
use lutra_text::{Buffer, ColumnUnit, Edit, LineEnding, Position, TextError};
use proptest::prelude::*;
const UNITS: [ColumnUnit; 2] = [ColumnUnit::Utf16, ColumnUnit::Char];
/// Text built from pieces that stress positions: ASCII, all three line
/// breaks, emoji (astral, ZWJ sequences, flags), CJK, combining marks,
/// Unicode separators that are not line breaks, and the highest code point.
fn tricky_text() -> impl Strategy<Value = String> {
let piece = prop_oneof![
Just("a"),
Just("Z"),
Just(" "),
Just("\t"),
Just("\n"),
Just("\r\n"),
Just("\r"),
Just("😀"),
Just("👨\u{200d}👩\u{200d}👧"),
Just("🇩🇪"),
Just("漢"),
Just("字"),
Just("e\u{301}"),
Just("\u{301}"),
Just("\u{2028}"),
Just("\u{85}"),
Just("\u{10ffff}"),
Just("é"),
];
prop::collection::vec(piece, 0..40).prop_map(|pieces| pieces.concat())
}
/// Tricky text or fully random strings.
fn any_text() -> impl Strategy<Value = String> {
prop_oneof![3 => tricky_text(), 1 => any::<String>()]
}
/// Edit ranges anywhere, including past the end and reversed.
fn any_edit(max: usize) -> impl Strategy<Value = Edit> {
(0..=max + 3, 0..=max + 3, tricky_text())
.prop_map(|(start, end, text)| Edit::replace(start..end, text))
}
/// The line breaks of `text` as byte ranges, CRLF as one break.
fn line_breaks(text: &str) -> Vec<std::ops::Range<usize>> {
let bytes = text.as_bytes();
let mut breaks = Vec::new();
let mut i = 0;
while i < bytes.len() {
match bytes[i] {
b'\r' if bytes.get(i + 1) == Some(&b'\n') => {
breaks.push(i..i + 2);
i += 2;
}
b'\r' | b'\n' => {
breaks.push(i..i + 1);
i += 1;
}
_ => i += 1,
}
}
breaks
}
/// Reference position of a byte offset, computed with `str` methods.
fn model_position(text: &str, offset: usize, unit: ColumnUnit) -> Option<Position> {
if !text.is_char_boundary(offset) {
return None;
}
let breaks = line_breaks(text);
if breaks.iter().any(|b| b.start < offset && offset < b.end) {
return None;
}
let line = breaks.iter().filter(|b| b.end <= offset).count();
let line_start = breaks
.iter()
.filter(|b| b.end <= offset)
.map(|b| b.end)
.next_back()
.unwrap_or(0);
let prefix = &text[line_start..offset];
let column = match unit {
ColumnUnit::Utf16 => prefix.encode_utf16().count(),
ColumnUnit::Char => prefix.chars().count(),
};
Some(Position::new(line, column))
}
/// Applies edits to a `String` with the documented batch semantics, or
/// `None` where the buffer must reject the batch.
fn model_apply(text: &str, edits: &[Edit]) -> Option<String> {
let mut sorted: Vec<&Edit> = edits.iter().collect();
for e in &sorted {
if e.range.start > e.range.end
|| !text.is_char_boundary(e.range.start)
|| !text.is_char_boundary(e.range.end)
{
return None;
}
}
sorted.sort_by_key(|e| (e.range.start, e.range.end));
if sorted.windows(2).any(|w| w[0].range.end > w[1].range.start) {
return None;
}
let mut out = String::new();
let mut at = 0;
for e in sorted {
out.push_str(&text[at..e.range.start]);
out.push_str(&e.text);
at = e.range.end;
}
out.push_str(&text[at..]);
Some(out)
}
proptest! {
#![proptest_config(ProptestConfig::with_cases(512))]
#[test]
fn counts_match_the_string(text in any_text()) {
let b = Buffer::new(&text);
prop_assert_eq!(b.text(), text.clone());
prop_assert_eq!(b.len_bytes(), text.len());
prop_assert_eq!(b.len_chars(), text.chars().count());
prop_assert_eq!(b.len_utf16(), text.encode_utf16().count());
prop_assert_eq!(b.line_count(), line_breaks(&text).len() + 1);
}
#[test]
fn every_byte_offset_converts_like_the_model_and_round_trips(text in any_text()) {
let b = Buffer::new(&text);
for offset in 0..=text.len() + 2 {
for unit in UNITS {
let got = b.offset_to_position(offset, unit);
prop_assert_eq!(got.as_ref().ok().copied(), model_position(&text, offset, unit));
if let Ok(position) = got {
prop_assert_eq!(b.position_to_offset(position, unit), Ok(offset));
}
}
let utf16 = b.offset_to_utf16(offset);
prop_assert_eq!(utf16.is_ok(), text.is_char_boundary(offset));
if let Ok(cu) = utf16 {
prop_assert_eq!(cu, text[..offset].encode_utf16().count());
prop_assert_eq!(b.utf16_to_offset(cu), Ok(offset));
}
}
}
#[test]
fn random_positions_never_panic_and_round_trip(
text in any_text(),
line in 0usize..12,
column in 0usize..48,
) {
let b = Buffer::new(&text);
for unit in UNITS {
if let Ok(offset) = b.position_to_offset(Position::new(line, column), unit) {
prop_assert_eq!(
b.offset_to_position(offset, unit),
Ok(Position::new(line, column))
);
}
}
for cu in 0..=b.len_utf16() + 2 {
if let Ok(offset) = b.utf16_to_offset(cu) {
prop_assert_eq!(b.offset_to_utf16(offset), Ok(cu));
}
}
}
#[test]
fn random_edit_batches_never_panic_and_match_the_model(
text in tricky_text(),
edits in prop::collection::vec(any_edit(160), 0..5),
) {
let mut b = Buffer::new(&text);
let expected = model_apply(&text, &edits);
let result = b.apply(&edits);
prop_assert_eq!(result.is_ok(), expected.is_some(), "{:?}", result);
prop_assert_eq!(b.text(), expected.unwrap_or(text));
prop_assert_eq!(b.len_bytes(), b.text().len());
prop_assert_eq!(b.line_count(), line_breaks(&b.text()).len() + 1);
}
#[test]
fn sequences_of_single_edits_stay_consistent(
text in tricky_text(),
edits in prop::collection::vec(any_edit(80), 0..20),
) {
let mut b = Buffer::new(&text);
let mut model = text;
for edit in edits {
let expected = model_apply(&model, std::slice::from_ref(&edit));
let result = b.apply(std::slice::from_ref(&edit));
prop_assert_eq!(result.is_ok(), expected.is_some());
if let Some(next) = expected {
model = next;
}
prop_assert_eq!(b.text(), model.clone());
}
for line in 0..b.line_count() {
let range = b.line_range(line).unwrap();
for unit in UNITS {
let start = b.offset_to_position(range.start, unit).unwrap();
prop_assert_eq!(start, Position::new(line, 0));
let end = b.offset_to_position(range.end, unit).unwrap();
prop_assert_eq!(b.position_to_offset(end, unit), Ok(range.end));
let past_end = Position::new(line, end.column + 1);
let is_out_of_range = matches!(
b.position_to_offset(past_end, unit),
Err(TextError::ColumnOutOfRange { .. })
);
prop_assert!(is_out_of_range);
}
}
}
#[test]
fn save_text_uses_only_the_buffers_line_ending(text in tricky_text()) {
let b = Buffer::new(&text);
let ending = b.line_ending();
prop_assert_eq!(Some(ending), LineEnding::detect(&text).or(Some(LineEnding::Lf)));
let saved = b.save_text();
let breaks = line_breaks(&saved);
prop_assert_eq!(breaks.len(), line_breaks(&text).len());
for range in breaks {
prop_assert_eq!(&saved[range], ending.as_str());
}
prop_assert_eq!(Buffer::new(&saved).text(), saved);
}
}