Lutra/crates/text/src/buffer.rs
danny 5dac918b14 M1.1: text buffer with UTF-16 positions, edit batches and line endings
lutra-text gets Buffer on a ropey rope: byte/char/UTF-16/line counts,
byte offset <-> (line, column) in UTF-16 code units or chars, absolute
UTF-16 offsets, atomic Edit batches in LSP TextEdit[] semantics, and
\n / \r\n / \r detection with a keep-the-file's-ending save policy.
All conversions return TextError instead of panicking.

ropey 1.6.1 (MIT) without unicode_lines, so only the LSP/CodeMirror
line breaks split lines. proptest moves into workspace deps.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-14 19:20:18 +02:00

573 lines
19 KiB
Rust

//! The rope-backed text buffer.
use std::ops::Range;
use ropey::{Rope, RopeSlice};
use crate::{ColumnUnit, Edit, LineEnding, Position, TextError};
/// An editable text held in a rope, with the line ending of its file.
///
/// Offsets are byte offsets into the UTF-8 text. Every conversion checks
/// its input and returns a [`TextError`] for offsets past the end, inside a
/// character or inside a CRLF line break, so offsets from untrusted sources
/// can be passed straight in. Line breaks are `\n`, `\r\n` and a lone `\r`
/// (see [`crate::line_ending`]); the line count is the number of line
/// breaks plus one, so a text ending in a line break has an empty last line.
#[derive(Debug, Clone, Default)]
pub struct Buffer {
/// The text.
rope: Rope,
/// The line ending used when saving.
line_ending: LineEnding,
}
impl Buffer {
/// Builds a buffer from `text`, taking the line ending detected in it or
/// [`LineEnding::Lf`] if the text has no line break.
pub fn new(text: &str) -> Self {
Self {
rope: Rope::from_str(text),
line_ending: LineEnding::detect(text).unwrap_or_default(),
}
}
/// The whole text as a string.
pub fn text(&self) -> String {
self.rope.to_string()
}
/// The text in the byte `range`.
///
/// # Errors
/// Fails when the range is reversed, reaches past the end or does not
/// lie on character boundaries.
pub fn slice(&self, range: Range<usize>) -> Result<String, TextError> {
let chars = self.char_range(&range)?;
Ok(self
.rope
.get_slice(chars)
.map(String::from)
.unwrap_or_default())
}
/// Length of the text in bytes.
pub fn len_bytes(&self) -> usize {
self.rope.len_bytes()
}
/// Length of the text in chars (Unicode scalar values).
pub fn len_chars(&self) -> usize {
self.rope.len_chars()
}
/// Length of the text in UTF-16 code units.
pub fn len_utf16(&self) -> usize {
self.rope.len_utf16_cu()
}
/// Number of lines: the number of line breaks plus one.
pub fn line_count(&self) -> usize {
self.rope.len_lines()
}
/// Whether the text is empty.
pub fn is_empty(&self) -> bool {
self.rope.len_bytes() == 0
}
/// The line ending the text is saved with.
pub fn line_ending(&self) -> LineEnding {
self.line_ending
}
/// Changes the line ending the text is saved with.
pub fn set_line_ending(&mut self, line_ending: LineEnding) {
self.line_ending = line_ending;
}
/// The text to write to disk: every line break replaced by the buffer's
/// line ending, so the file keeps its ending.
pub fn save_text(&self) -> String {
self.line_ending.normalize_chunks(self.rope.chunks())
}
/// The byte range of `line` without its line break.
///
/// # Errors
/// [`TextError::LineOutOfRange`] when the line does not exist.
pub fn line_range(&self, line: usize) -> Result<Range<usize>, TextError> {
let slice = self.rope.get_line(line).ok_or(TextError::LineOutOfRange {
line,
line_count: self.line_count(),
})?;
let start = self.rope.try_line_to_byte(line)?;
let content = slice.len_bytes().saturating_sub(line_break_len(&slice));
Ok(start..start.saturating_add(content))
}
/// The text of `line` without its line break.
///
/// # Errors
/// [`TextError::LineOutOfRange`] when the line does not exist.
pub fn line_text(&self, line: usize) -> Result<String, TextError> {
let range = self.line_range(line)?;
self.slice(range)
}
/// Converts a byte offset to a line and a column counted in `unit`.
///
/// # Errors
/// Fails when the offset is past the end, inside a character or between
/// the `\r` and `\n` of a CRLF line break.
pub fn offset_to_position(
&self,
offset: usize,
unit: ColumnUnit,
) -> Result<Position, TextError> {
let char_idx = self.offset_to_char(offset)?;
let line = self.rope.try_char_to_line(char_idx)?;
let range = self.line_range(line)?;
if offset > range.end {
return Err(TextError::InsideLineBreak { offset });
}
let line_start = self.rope.try_byte_to_char(range.start)?;
let column = match unit {
ColumnUnit::Char => char_idx.saturating_sub(line_start),
ColumnUnit::Utf16 => self
.rope
.try_char_to_utf16_cu(char_idx)?
.saturating_sub(self.rope.try_char_to_utf16_cu(line_start)?),
};
Ok(Position { line, column })
}
/// Converts a line and a column counted in `unit` to a byte offset.
///
/// # Errors
/// Fails when the line does not exist, the column is past the end of
/// the line (line break excluded) or, in UTF-16, falls inside a
/// surrogate pair.
pub fn position_to_offset(
&self,
position: Position,
unit: ColumnUnit,
) -> Result<usize, TextError> {
let Position { line, column } = position;
let range = self.line_range(line)?;
let start_char = self.rope.try_byte_to_char(range.start)?;
let end_char = self.rope.try_byte_to_char(range.end)?;
let char_idx = match unit {
ColumnUnit::Char => {
let line_len = end_char.saturating_sub(start_char);
if column > line_len {
return Err(TextError::ColumnOutOfRange {
line,
column,
line_len,
});
}
start_char.saturating_add(column)
}
ColumnUnit::Utf16 => {
let start_cu = self.rope.try_char_to_utf16_cu(start_char)?;
let end_cu = self.rope.try_char_to_utf16_cu(end_char)?;
let line_len = end_cu.saturating_sub(start_cu);
if column > line_len {
return Err(TextError::ColumnOutOfRange {
line,
column,
line_len,
});
}
let target = start_cu.saturating_add(column);
let char_idx = self.rope.try_utf16_cu_to_char(target)?;
if self.rope.try_char_to_utf16_cu(char_idx)? != target {
return Err(TextError::ColumnInsideChar { line, column });
}
char_idx
}
};
Ok(self.rope.try_char_to_byte(char_idx)?)
}
/// Converts a byte offset to an offset in UTF-16 code units from the
/// start of the text. Line breaks count as stored (`\r\n` is two).
///
/// # Errors
/// Fails when the offset is past the end or inside a character.
pub fn offset_to_utf16(&self, offset: usize) -> Result<usize, TextError> {
let char_idx = self.offset_to_char(offset)?;
Ok(self.rope.try_char_to_utf16_cu(char_idx)?)
}
/// Converts an offset in UTF-16 code units from the start of the text to
/// a byte offset.
///
/// # Errors
/// Fails when the offset is past the end or inside a surrogate pair.
pub fn utf16_to_offset(&self, offset: usize) -> Result<usize, TextError> {
let len = self.len_utf16();
if offset > len {
return Err(TextError::Utf16OutOfRange { offset, len });
}
let char_idx = self.rope.try_utf16_cu_to_char(offset)?;
if self.rope.try_char_to_utf16_cu(char_idx)? != offset {
return Err(TextError::Utf16InsideChar { offset });
}
Ok(self.rope.try_char_to_byte(char_idx)?)
}
/// Applies a batch of edits atomically.
///
/// All ranges refer to the text before the call, as in an LSP
/// `TextEdit[]`, and may come in any order. Insertions at the same
/// offset land in the order given, and an insertion at the start of a
/// replaced range lands before the replacement.
///
/// # Errors
/// Rejects the whole batch, leaving the buffer unchanged, when a range
/// is reversed, reaches past the end, does not lie on character
/// boundaries, or overlaps another edit of the batch.
pub fn apply(&mut self, edits: &[Edit]) -> Result<(), TextError> {
let mut resolved = Vec::with_capacity(edits.len());
for edit in edits {
let chars = self.char_range(&edit.range)?;
resolved.push((edit.range.clone(), chars, edit.text.as_str()));
}
resolved.sort_by_key(|(bytes, _, _)| (bytes.start, bytes.end));
for pair in resolved.windows(2) {
if let [(first, _, _), (second, _, _)] = pair
&& first.end > second.start
{
return Err(TextError::OverlappingEdits {
first: first.clone(),
second: second.clone(),
});
}
}
let mut rope = self.rope.clone();
for (_, chars, text) in resolved.iter().rev() {
rope.try_remove(chars.clone())?;
rope.try_insert(chars.start, text)?;
}
self.rope = rope;
Ok(())
}
/// Converts a byte offset on a character boundary to a char index.
fn offset_to_char(&self, offset: usize) -> Result<usize, TextError> {
let len = self.len_bytes();
if offset > len {
return Err(TextError::OffsetOutOfRange { offset, len });
}
let char_idx = self.rope.try_byte_to_char(offset)?;
if self.rope.try_char_to_byte(char_idx)? != offset {
return Err(TextError::NotCharBoundary { offset });
}
Ok(char_idx)
}
/// Converts a byte range on character boundaries to a char range.
fn char_range(&self, range: &Range<usize>) -> Result<Range<usize>, TextError> {
if range.start > range.end {
return Err(TextError::InvalidRange {
start: range.start,
end: range.end,
});
}
let end = self.offset_to_char(range.end)?;
let start = self.offset_to_char(range.start)?;
Ok(start..end)
}
}
impl From<&str> for Buffer {
fn from(text: &str) -> Self {
Self::new(text)
}
}
/// Length in bytes of the line break at the end of `line`: 2 for `\r\n`,
/// 1 for `\n` or `\r`, 0 for the last line.
fn line_break_len(line: &RopeSlice<'_>) -> usize {
let len = line.len_chars();
let last = len.checked_sub(1).and_then(|i| line.get_char(i));
let before_last = len.checked_sub(2).and_then(|i| line.get_char(i));
match (before_last, last) {
(Some('\r'), Some('\n')) => 2,
(_, Some('\n' | '\r')) => 1,
_ => 0,
}
}
#[cfg(test)]
mod tests {
use super::*;
const UNITS: [ColumnUnit; 2] = [ColumnUnit::Utf16, ColumnUnit::Char];
fn pos(line: usize, column: usize) -> Position {
Position::new(line, column)
}
#[test]
fn counts_bytes_chars_utf16_and_lines() {
let b = Buffer::new("a😀\n漢e\u{301}\r\nz");
assert_eq!(b.len_bytes(), "a😀\n漢e\u{301}\r\nz".len());
assert_eq!(b.len_chars(), 9);
assert_eq!(b.len_utf16(), 10);
assert_eq!(b.line_count(), 3);
assert_eq!(b.text(), "a😀\n漢e\u{301}\r\nz");
assert!(!b.is_empty());
}
#[test]
fn empty_text_has_one_empty_line() {
let b = Buffer::new("");
assert!(b.is_empty());
assert_eq!(b.line_count(), 1);
assert_eq!(b.line_range(0), Ok(0..0));
assert_eq!(b.offset_to_position(0, ColumnUnit::Utf16), Ok(pos(0, 0)));
assert_eq!(b.position_to_offset(pos(0, 0), ColumnUnit::Char), Ok(0));
}
#[test]
fn trailing_line_break_adds_an_empty_last_line() {
let b = Buffer::new("a\n");
assert_eq!(b.line_count(), 2);
assert_eq!(b.line_range(1), Ok(2..2));
assert_eq!(b.offset_to_position(2, ColumnUnit::Char), Ok(pos(1, 0)));
}
#[test]
fn all_three_line_breaks_split_lines_and_unicode_separators_do_not() {
let b = Buffer::new("a\nb\r\nc\rd\u{2028}e\u{85}f");
assert_eq!(b.line_count(), 4);
assert_eq!(b.line_text(0).unwrap(), "a");
assert_eq!(b.line_text(1).unwrap(), "b");
assert_eq!(b.line_text(2).unwrap(), "c");
assert_eq!(b.line_text(3).unwrap(), "d\u{2028}e\u{85}f");
}
#[test]
fn columns_differ_by_unit_for_astral_chars() {
let b = Buffer::new("x\n😀a");
let after_a = "x\n😀a".len();
assert_eq!(
b.offset_to_position(after_a, ColumnUnit::Utf16),
Ok(pos(1, 3))
);
assert_eq!(
b.offset_to_position(after_a, ColumnUnit::Char),
Ok(pos(1, 2))
);
assert_eq!(
b.position_to_offset(pos(1, 3), ColumnUnit::Utf16),
Ok(after_a)
);
assert_eq!(
b.position_to_offset(pos(1, 2), ColumnUnit::Char),
Ok(after_a)
);
}
#[test]
fn combining_marks_and_cjk_count_one_char_each() {
let b = Buffer::new("e\u{301}漢");
assert_eq!(b.offset_to_position(3, ColumnUnit::Char), Ok(pos(0, 2)));
assert_eq!(b.offset_to_position(6, ColumnUnit::Utf16), Ok(pos(0, 3)));
}
#[test]
fn offsets_inside_chars_or_past_the_end_are_errors() {
let b = Buffer::new("😀");
for unit in UNITS {
assert_eq!(
b.offset_to_position(1, unit),
Err(TextError::NotCharBoundary { offset: 1 })
);
assert_eq!(
b.offset_to_position(5, unit),
Err(TextError::OffsetOutOfRange { offset: 5, len: 4 })
);
assert!(b.offset_to_position(usize::MAX, unit).is_err());
}
}
#[test]
fn offset_inside_crlf_is_an_error() {
let b = Buffer::new("a\r\nb");
assert_eq!(b.offset_to_position(1, ColumnUnit::Char), Ok(pos(0, 1)));
assert_eq!(
b.offset_to_position(2, ColumnUnit::Char),
Err(TextError::InsideLineBreak { offset: 2 })
);
assert_eq!(b.offset_to_position(3, ColumnUnit::Char), Ok(pos(1, 0)));
}
#[test]
fn bad_positions_are_errors() {
let b = Buffer::new("😀\nab");
assert_eq!(
b.position_to_offset(pos(2, 0), ColumnUnit::Char),
Err(TextError::LineOutOfRange {
line: 2,
line_count: 2
})
);
assert_eq!(
b.position_to_offset(pos(0, 2), ColumnUnit::Char),
Err(TextError::ColumnOutOfRange {
line: 0,
column: 2,
line_len: 1
})
);
assert_eq!(
b.position_to_offset(pos(0, 1), ColumnUnit::Utf16),
Err(TextError::ColumnInsideChar { line: 0, column: 1 })
);
assert_eq!(b.position_to_offset(pos(0, 2), ColumnUnit::Utf16), Ok(4));
assert!(
b.position_to_offset(pos(usize::MAX, usize::MAX), ColumnUnit::Utf16)
.is_err()
);
assert!(
b.position_to_offset(pos(1, usize::MAX), ColumnUnit::Char)
.is_err()
);
}
#[test]
fn absolute_utf16_offsets() {
let b = Buffer::new("a😀b");
assert_eq!(b.offset_to_utf16(5), Ok(3));
assert_eq!(b.utf16_to_offset(3), Ok(5));
assert_eq!(
b.utf16_to_offset(2),
Err(TextError::Utf16InsideChar { offset: 2 })
);
assert_eq!(
b.utf16_to_offset(5),
Err(TextError::Utf16OutOfRange { offset: 5, len: 4 })
);
assert_eq!(
b.offset_to_utf16(2),
Err(TextError::NotCharBoundary { offset: 2 })
);
}
#[test]
fn slice_checks_its_range() {
let b = Buffer::new("héllo");
assert_eq!(b.slice(1..3).unwrap(), "é");
assert_eq!(b.slice(2..3), Err(TextError::NotCharBoundary { offset: 2 }));
assert_eq!(
b.slice(Range { start: 3, end: 1 }),
Err(TextError::InvalidRange { start: 3, end: 1 })
);
assert!(b.slice(0..99).is_err());
}
#[test]
fn apply_insert_delete_replace() {
let mut b = Buffer::new("hello world");
b.apply(&[Edit::replace(0..5, "goodbye")]).unwrap();
assert_eq!(b.text(), "goodbye world");
b.apply(&[Edit::insert(13, "!")]).unwrap();
assert_eq!(b.text(), "goodbye world!");
b.apply(&[Edit::delete(7..13)]).unwrap();
assert_eq!(b.text(), "goodbye!");
b.apply(&[]).unwrap();
assert_eq!(b.text(), "goodbye!");
}
#[test]
fn batch_ranges_refer_to_the_original_text_in_any_order() {
let mut b = Buffer::new("abcdef");
b.apply(&[
Edit::replace(4..6, "EF"),
Edit::replace(0..1, "AA"),
Edit::delete(2..3),
])
.unwrap();
assert_eq!(b.text(), "AAbdEF");
}
#[test]
fn insertions_at_one_offset_keep_their_order() {
let mut b = Buffer::new("ab");
b.apply(&[
Edit::insert(1, "1"),
Edit::replace(1..2, "B"),
Edit::insert(1, "2"),
])
.unwrap();
assert_eq!(b.text(), "a12B");
}
#[test]
fn overlapping_edits_are_rejected_and_leave_the_buffer_unchanged() {
let mut b = Buffer::new("abcdef");
assert_eq!(
b.apply(&[Edit::delete(3..5), Edit::delete(1..4)]),
Err(TextError::OverlappingEdits {
first: 1..4,
second: 3..5
})
);
assert_eq!(b.text(), "abcdef");
assert!(
b.apply(&[Edit::delete(1..4), Edit::insert(2, "x")])
.is_err()
);
assert_eq!(b.text(), "abcdef");
}
#[test]
fn invalid_edits_are_rejected_and_leave_the_buffer_unchanged() {
let mut b = Buffer::new("a😀");
let bad = [
Edit::insert(2, "x"),
Edit::delete(0..9),
Edit::replace(Range { start: 1, end: 0 }, "x"),
Edit::insert(usize::MAX, "x"),
];
for edit in bad {
assert!(b.apply(&[Edit::insert(0, "ok"), edit]).is_err());
assert_eq!(b.text(), "a😀");
}
}
#[test]
fn edits_may_split_and_join_crlf() {
let mut b = Buffer::new("a\r\nb");
b.apply(&[Edit::insert(2, "x")]).unwrap();
assert_eq!(b.line_count(), 3);
b.apply(&[Edit::delete(2..3)]).unwrap();
assert_eq!(b.line_count(), 2);
assert_eq!(b.line_text(1).unwrap(), "b");
}
#[test]
fn keeps_the_files_line_ending_on_save() {
let mut crlf = Buffer::new("a\r\nb\r\n");
assert_eq!(crlf.line_ending(), LineEnding::CrLf);
crlf.apply(&[Edit::insert(1, "\nnew")]).unwrap();
assert_eq!(crlf.save_text(), "a\r\nnew\r\nb\r\n");
let mut cr = Buffer::new("a\rb");
assert_eq!(cr.line_ending(), LineEnding::Cr);
cr.apply(&[Edit::insert(3, "\r\nc")]).unwrap();
assert_eq!(cr.save_text(), "a\rb\rc");
let plain = Buffer::new("no break");
assert_eq!(plain.line_ending(), LineEnding::Lf);
let mut lf = Buffer::new("a\nb");
lf.set_line_ending(LineEnding::CrLf);
assert_eq!(lf.save_text(), "a\r\nb");
assert_eq!(lf.text(), "a\nb");
}
}