M1.1: text buffer with UTF-16 positions, edit batches and line endings
lutra-text gets Buffer on a ropey rope: byte/char/UTF-16/line counts, byte offset <-> (line, column) in UTF-16 code units or chars, absolute UTF-16 offsets, atomic Edit batches in LSP TextEdit[] semantics, and \n / \r\n / \r detection with a keep-the-file's-ending save policy. All conversions return TextError instead of panicking. ropey 1.6.1 (MIT) without unicode_lines, so only the LSP/CodeMirror line breaks split lines. proptest moves into workspace deps. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
faadb05eb3
commit
5dac918b14
12 changed files with 1278 additions and 3 deletions
573
crates/text/src/buffer.rs
Normal file
573
crates/text/src/buffer.rs
Normal file
|
|
@ -0,0 +1,573 @@
|
|||
//! The rope-backed text buffer.
|
||||
|
||||
use std::ops::Range;
|
||||
|
||||
use ropey::{Rope, RopeSlice};
|
||||
|
||||
use crate::{ColumnUnit, Edit, LineEnding, Position, TextError};
|
||||
|
||||
/// An editable text held in a rope, with the line ending of its file.
|
||||
///
|
||||
/// Offsets are byte offsets into the UTF-8 text. Every conversion checks
|
||||
/// its input and returns a [`TextError`] for offsets past the end, inside a
|
||||
/// character or inside a CRLF line break, so offsets from untrusted sources
|
||||
/// can be passed straight in. Line breaks are `\n`, `\r\n` and a lone `\r`
|
||||
/// (see [`crate::line_ending`]); the line count is the number of line
|
||||
/// breaks plus one, so a text ending in a line break has an empty last line.
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub struct Buffer {
|
||||
/// The text.
|
||||
rope: Rope,
|
||||
/// The line ending used when saving.
|
||||
line_ending: LineEnding,
|
||||
}
|
||||
|
||||
impl Buffer {
|
||||
/// Builds a buffer from `text`, taking the line ending detected in it or
|
||||
/// [`LineEnding::Lf`] if the text has no line break.
|
||||
pub fn new(text: &str) -> Self {
|
||||
Self {
|
||||
rope: Rope::from_str(text),
|
||||
line_ending: LineEnding::detect(text).unwrap_or_default(),
|
||||
}
|
||||
}
|
||||
|
||||
/// The whole text as a string.
|
||||
pub fn text(&self) -> String {
|
||||
self.rope.to_string()
|
||||
}
|
||||
|
||||
/// The text in the byte `range`.
|
||||
///
|
||||
/// # Errors
|
||||
/// Fails when the range is reversed, reaches past the end or does not
|
||||
/// lie on character boundaries.
|
||||
pub fn slice(&self, range: Range<usize>) -> Result<String, TextError> {
|
||||
let chars = self.char_range(&range)?;
|
||||
Ok(self
|
||||
.rope
|
||||
.get_slice(chars)
|
||||
.map(String::from)
|
||||
.unwrap_or_default())
|
||||
}
|
||||
|
||||
/// Length of the text in bytes.
|
||||
pub fn len_bytes(&self) -> usize {
|
||||
self.rope.len_bytes()
|
||||
}
|
||||
|
||||
/// Length of the text in chars (Unicode scalar values).
|
||||
pub fn len_chars(&self) -> usize {
|
||||
self.rope.len_chars()
|
||||
}
|
||||
|
||||
/// Length of the text in UTF-16 code units.
|
||||
pub fn len_utf16(&self) -> usize {
|
||||
self.rope.len_utf16_cu()
|
||||
}
|
||||
|
||||
/// Number of lines: the number of line breaks plus one.
|
||||
pub fn line_count(&self) -> usize {
|
||||
self.rope.len_lines()
|
||||
}
|
||||
|
||||
/// Whether the text is empty.
|
||||
pub fn is_empty(&self) -> bool {
|
||||
self.rope.len_bytes() == 0
|
||||
}
|
||||
|
||||
/// The line ending the text is saved with.
|
||||
pub fn line_ending(&self) -> LineEnding {
|
||||
self.line_ending
|
||||
}
|
||||
|
||||
/// Changes the line ending the text is saved with.
|
||||
pub fn set_line_ending(&mut self, line_ending: LineEnding) {
|
||||
self.line_ending = line_ending;
|
||||
}
|
||||
|
||||
/// The text to write to disk: every line break replaced by the buffer's
|
||||
/// line ending, so the file keeps its ending.
|
||||
pub fn save_text(&self) -> String {
|
||||
self.line_ending.normalize_chunks(self.rope.chunks())
|
||||
}
|
||||
|
||||
/// The byte range of `line` without its line break.
|
||||
///
|
||||
/// # Errors
|
||||
/// [`TextError::LineOutOfRange`] when the line does not exist.
|
||||
pub fn line_range(&self, line: usize) -> Result<Range<usize>, TextError> {
|
||||
let slice = self.rope.get_line(line).ok_or(TextError::LineOutOfRange {
|
||||
line,
|
||||
line_count: self.line_count(),
|
||||
})?;
|
||||
let start = self.rope.try_line_to_byte(line)?;
|
||||
let content = slice.len_bytes().saturating_sub(line_break_len(&slice));
|
||||
Ok(start..start.saturating_add(content))
|
||||
}
|
||||
|
||||
/// The text of `line` without its line break.
|
||||
///
|
||||
/// # Errors
|
||||
/// [`TextError::LineOutOfRange`] when the line does not exist.
|
||||
pub fn line_text(&self, line: usize) -> Result<String, TextError> {
|
||||
let range = self.line_range(line)?;
|
||||
self.slice(range)
|
||||
}
|
||||
|
||||
/// Converts a byte offset to a line and a column counted in `unit`.
|
||||
///
|
||||
/// # Errors
|
||||
/// Fails when the offset is past the end, inside a character or between
|
||||
/// the `\r` and `\n` of a CRLF line break.
|
||||
pub fn offset_to_position(
|
||||
&self,
|
||||
offset: usize,
|
||||
unit: ColumnUnit,
|
||||
) -> Result<Position, TextError> {
|
||||
let char_idx = self.offset_to_char(offset)?;
|
||||
let line = self.rope.try_char_to_line(char_idx)?;
|
||||
let range = self.line_range(line)?;
|
||||
if offset > range.end {
|
||||
return Err(TextError::InsideLineBreak { offset });
|
||||
}
|
||||
let line_start = self.rope.try_byte_to_char(range.start)?;
|
||||
let column = match unit {
|
||||
ColumnUnit::Char => char_idx.saturating_sub(line_start),
|
||||
ColumnUnit::Utf16 => self
|
||||
.rope
|
||||
.try_char_to_utf16_cu(char_idx)?
|
||||
.saturating_sub(self.rope.try_char_to_utf16_cu(line_start)?),
|
||||
};
|
||||
Ok(Position { line, column })
|
||||
}
|
||||
|
||||
/// Converts a line and a column counted in `unit` to a byte offset.
|
||||
///
|
||||
/// # Errors
|
||||
/// Fails when the line does not exist, the column is past the end of
|
||||
/// the line (line break excluded) or, in UTF-16, falls inside a
|
||||
/// surrogate pair.
|
||||
pub fn position_to_offset(
|
||||
&self,
|
||||
position: Position,
|
||||
unit: ColumnUnit,
|
||||
) -> Result<usize, TextError> {
|
||||
let Position { line, column } = position;
|
||||
let range = self.line_range(line)?;
|
||||
let start_char = self.rope.try_byte_to_char(range.start)?;
|
||||
let end_char = self.rope.try_byte_to_char(range.end)?;
|
||||
let char_idx = match unit {
|
||||
ColumnUnit::Char => {
|
||||
let line_len = end_char.saturating_sub(start_char);
|
||||
if column > line_len {
|
||||
return Err(TextError::ColumnOutOfRange {
|
||||
line,
|
||||
column,
|
||||
line_len,
|
||||
});
|
||||
}
|
||||
start_char.saturating_add(column)
|
||||
}
|
||||
ColumnUnit::Utf16 => {
|
||||
let start_cu = self.rope.try_char_to_utf16_cu(start_char)?;
|
||||
let end_cu = self.rope.try_char_to_utf16_cu(end_char)?;
|
||||
let line_len = end_cu.saturating_sub(start_cu);
|
||||
if column > line_len {
|
||||
return Err(TextError::ColumnOutOfRange {
|
||||
line,
|
||||
column,
|
||||
line_len,
|
||||
});
|
||||
}
|
||||
let target = start_cu.saturating_add(column);
|
||||
let char_idx = self.rope.try_utf16_cu_to_char(target)?;
|
||||
if self.rope.try_char_to_utf16_cu(char_idx)? != target {
|
||||
return Err(TextError::ColumnInsideChar { line, column });
|
||||
}
|
||||
char_idx
|
||||
}
|
||||
};
|
||||
Ok(self.rope.try_char_to_byte(char_idx)?)
|
||||
}
|
||||
|
||||
/// Converts a byte offset to an offset in UTF-16 code units from the
|
||||
/// start of the text. Line breaks count as stored (`\r\n` is two).
|
||||
///
|
||||
/// # Errors
|
||||
/// Fails when the offset is past the end or inside a character.
|
||||
pub fn offset_to_utf16(&self, offset: usize) -> Result<usize, TextError> {
|
||||
let char_idx = self.offset_to_char(offset)?;
|
||||
Ok(self.rope.try_char_to_utf16_cu(char_idx)?)
|
||||
}
|
||||
|
||||
/// Converts an offset in UTF-16 code units from the start of the text to
|
||||
/// a byte offset.
|
||||
///
|
||||
/// # Errors
|
||||
/// Fails when the offset is past the end or inside a surrogate pair.
|
||||
pub fn utf16_to_offset(&self, offset: usize) -> Result<usize, TextError> {
|
||||
let len = self.len_utf16();
|
||||
if offset > len {
|
||||
return Err(TextError::Utf16OutOfRange { offset, len });
|
||||
}
|
||||
let char_idx = self.rope.try_utf16_cu_to_char(offset)?;
|
||||
if self.rope.try_char_to_utf16_cu(char_idx)? != offset {
|
||||
return Err(TextError::Utf16InsideChar { offset });
|
||||
}
|
||||
Ok(self.rope.try_char_to_byte(char_idx)?)
|
||||
}
|
||||
|
||||
/// Applies a batch of edits atomically.
|
||||
///
|
||||
/// All ranges refer to the text before the call, as in an LSP
|
||||
/// `TextEdit[]`, and may come in any order. Insertions at the same
|
||||
/// offset land in the order given, and an insertion at the start of a
|
||||
/// replaced range lands before the replacement.
|
||||
///
|
||||
/// # Errors
|
||||
/// Rejects the whole batch, leaving the buffer unchanged, when a range
|
||||
/// is reversed, reaches past the end, does not lie on character
|
||||
/// boundaries, or overlaps another edit of the batch.
|
||||
pub fn apply(&mut self, edits: &[Edit]) -> Result<(), TextError> {
|
||||
let mut resolved = Vec::with_capacity(edits.len());
|
||||
for edit in edits {
|
||||
let chars = self.char_range(&edit.range)?;
|
||||
resolved.push((edit.range.clone(), chars, edit.text.as_str()));
|
||||
}
|
||||
resolved.sort_by_key(|(bytes, _, _)| (bytes.start, bytes.end));
|
||||
for pair in resolved.windows(2) {
|
||||
if let [(first, _, _), (second, _, _)] = pair
|
||||
&& first.end > second.start
|
||||
{
|
||||
return Err(TextError::OverlappingEdits {
|
||||
first: first.clone(),
|
||||
second: second.clone(),
|
||||
});
|
||||
}
|
||||
}
|
||||
let mut rope = self.rope.clone();
|
||||
for (_, chars, text) in resolved.iter().rev() {
|
||||
rope.try_remove(chars.clone())?;
|
||||
rope.try_insert(chars.start, text)?;
|
||||
}
|
||||
self.rope = rope;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Converts a byte offset on a character boundary to a char index.
|
||||
fn offset_to_char(&self, offset: usize) -> Result<usize, TextError> {
|
||||
let len = self.len_bytes();
|
||||
if offset > len {
|
||||
return Err(TextError::OffsetOutOfRange { offset, len });
|
||||
}
|
||||
let char_idx = self.rope.try_byte_to_char(offset)?;
|
||||
if self.rope.try_char_to_byte(char_idx)? != offset {
|
||||
return Err(TextError::NotCharBoundary { offset });
|
||||
}
|
||||
Ok(char_idx)
|
||||
}
|
||||
|
||||
/// Converts a byte range on character boundaries to a char range.
|
||||
fn char_range(&self, range: &Range<usize>) -> Result<Range<usize>, TextError> {
|
||||
if range.start > range.end {
|
||||
return Err(TextError::InvalidRange {
|
||||
start: range.start,
|
||||
end: range.end,
|
||||
});
|
||||
}
|
||||
let end = self.offset_to_char(range.end)?;
|
||||
let start = self.offset_to_char(range.start)?;
|
||||
Ok(start..end)
|
||||
}
|
||||
}
|
||||
|
||||
impl From<&str> for Buffer {
|
||||
fn from(text: &str) -> Self {
|
||||
Self::new(text)
|
||||
}
|
||||
}
|
||||
|
||||
/// Length in bytes of the line break at the end of `line`: 2 for `\r\n`,
|
||||
/// 1 for `\n` or `\r`, 0 for the last line.
|
||||
fn line_break_len(line: &RopeSlice<'_>) -> usize {
|
||||
let len = line.len_chars();
|
||||
let last = len.checked_sub(1).and_then(|i| line.get_char(i));
|
||||
let before_last = len.checked_sub(2).and_then(|i| line.get_char(i));
|
||||
match (before_last, last) {
|
||||
(Some('\r'), Some('\n')) => 2,
|
||||
(_, Some('\n' | '\r')) => 1,
|
||||
_ => 0,
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
const UNITS: [ColumnUnit; 2] = [ColumnUnit::Utf16, ColumnUnit::Char];
|
||||
|
||||
fn pos(line: usize, column: usize) -> Position {
|
||||
Position::new(line, column)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn counts_bytes_chars_utf16_and_lines() {
|
||||
let b = Buffer::new("a😀\n漢e\u{301}\r\nz");
|
||||
assert_eq!(b.len_bytes(), "a😀\n漢e\u{301}\r\nz".len());
|
||||
assert_eq!(b.len_chars(), 9);
|
||||
assert_eq!(b.len_utf16(), 10);
|
||||
assert_eq!(b.line_count(), 3);
|
||||
assert_eq!(b.text(), "a😀\n漢e\u{301}\r\nz");
|
||||
assert!(!b.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn empty_text_has_one_empty_line() {
|
||||
let b = Buffer::new("");
|
||||
assert!(b.is_empty());
|
||||
assert_eq!(b.line_count(), 1);
|
||||
assert_eq!(b.line_range(0), Ok(0..0));
|
||||
assert_eq!(b.offset_to_position(0, ColumnUnit::Utf16), Ok(pos(0, 0)));
|
||||
assert_eq!(b.position_to_offset(pos(0, 0), ColumnUnit::Char), Ok(0));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn trailing_line_break_adds_an_empty_last_line() {
|
||||
let b = Buffer::new("a\n");
|
||||
assert_eq!(b.line_count(), 2);
|
||||
assert_eq!(b.line_range(1), Ok(2..2));
|
||||
assert_eq!(b.offset_to_position(2, ColumnUnit::Char), Ok(pos(1, 0)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn all_three_line_breaks_split_lines_and_unicode_separators_do_not() {
|
||||
let b = Buffer::new("a\nb\r\nc\rd\u{2028}e\u{85}f");
|
||||
assert_eq!(b.line_count(), 4);
|
||||
assert_eq!(b.line_text(0).unwrap(), "a");
|
||||
assert_eq!(b.line_text(1).unwrap(), "b");
|
||||
assert_eq!(b.line_text(2).unwrap(), "c");
|
||||
assert_eq!(b.line_text(3).unwrap(), "d\u{2028}e\u{85}f");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn columns_differ_by_unit_for_astral_chars() {
|
||||
let b = Buffer::new("x\n😀a");
|
||||
let after_a = "x\n😀a".len();
|
||||
assert_eq!(
|
||||
b.offset_to_position(after_a, ColumnUnit::Utf16),
|
||||
Ok(pos(1, 3))
|
||||
);
|
||||
assert_eq!(
|
||||
b.offset_to_position(after_a, ColumnUnit::Char),
|
||||
Ok(pos(1, 2))
|
||||
);
|
||||
assert_eq!(
|
||||
b.position_to_offset(pos(1, 3), ColumnUnit::Utf16),
|
||||
Ok(after_a)
|
||||
);
|
||||
assert_eq!(
|
||||
b.position_to_offset(pos(1, 2), ColumnUnit::Char),
|
||||
Ok(after_a)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn combining_marks_and_cjk_count_one_char_each() {
|
||||
let b = Buffer::new("e\u{301}漢");
|
||||
assert_eq!(b.offset_to_position(3, ColumnUnit::Char), Ok(pos(0, 2)));
|
||||
assert_eq!(b.offset_to_position(6, ColumnUnit::Utf16), Ok(pos(0, 3)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn offsets_inside_chars_or_past_the_end_are_errors() {
|
||||
let b = Buffer::new("😀");
|
||||
for unit in UNITS {
|
||||
assert_eq!(
|
||||
b.offset_to_position(1, unit),
|
||||
Err(TextError::NotCharBoundary { offset: 1 })
|
||||
);
|
||||
assert_eq!(
|
||||
b.offset_to_position(5, unit),
|
||||
Err(TextError::OffsetOutOfRange { offset: 5, len: 4 })
|
||||
);
|
||||
assert!(b.offset_to_position(usize::MAX, unit).is_err());
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn offset_inside_crlf_is_an_error() {
|
||||
let b = Buffer::new("a\r\nb");
|
||||
assert_eq!(b.offset_to_position(1, ColumnUnit::Char), Ok(pos(0, 1)));
|
||||
assert_eq!(
|
||||
b.offset_to_position(2, ColumnUnit::Char),
|
||||
Err(TextError::InsideLineBreak { offset: 2 })
|
||||
);
|
||||
assert_eq!(b.offset_to_position(3, ColumnUnit::Char), Ok(pos(1, 0)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bad_positions_are_errors() {
|
||||
let b = Buffer::new("😀\nab");
|
||||
assert_eq!(
|
||||
b.position_to_offset(pos(2, 0), ColumnUnit::Char),
|
||||
Err(TextError::LineOutOfRange {
|
||||
line: 2,
|
||||
line_count: 2
|
||||
})
|
||||
);
|
||||
assert_eq!(
|
||||
b.position_to_offset(pos(0, 2), ColumnUnit::Char),
|
||||
Err(TextError::ColumnOutOfRange {
|
||||
line: 0,
|
||||
column: 2,
|
||||
line_len: 1
|
||||
})
|
||||
);
|
||||
assert_eq!(
|
||||
b.position_to_offset(pos(0, 1), ColumnUnit::Utf16),
|
||||
Err(TextError::ColumnInsideChar { line: 0, column: 1 })
|
||||
);
|
||||
assert_eq!(b.position_to_offset(pos(0, 2), ColumnUnit::Utf16), Ok(4));
|
||||
assert!(
|
||||
b.position_to_offset(pos(usize::MAX, usize::MAX), ColumnUnit::Utf16)
|
||||
.is_err()
|
||||
);
|
||||
assert!(
|
||||
b.position_to_offset(pos(1, usize::MAX), ColumnUnit::Char)
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn absolute_utf16_offsets() {
|
||||
let b = Buffer::new("a😀b");
|
||||
assert_eq!(b.offset_to_utf16(5), Ok(3));
|
||||
assert_eq!(b.utf16_to_offset(3), Ok(5));
|
||||
assert_eq!(
|
||||
b.utf16_to_offset(2),
|
||||
Err(TextError::Utf16InsideChar { offset: 2 })
|
||||
);
|
||||
assert_eq!(
|
||||
b.utf16_to_offset(5),
|
||||
Err(TextError::Utf16OutOfRange { offset: 5, len: 4 })
|
||||
);
|
||||
assert_eq!(
|
||||
b.offset_to_utf16(2),
|
||||
Err(TextError::NotCharBoundary { offset: 2 })
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn slice_checks_its_range() {
|
||||
let b = Buffer::new("héllo");
|
||||
assert_eq!(b.slice(1..3).unwrap(), "é");
|
||||
assert_eq!(b.slice(2..3), Err(TextError::NotCharBoundary { offset: 2 }));
|
||||
assert_eq!(
|
||||
b.slice(Range { start: 3, end: 1 }),
|
||||
Err(TextError::InvalidRange { start: 3, end: 1 })
|
||||
);
|
||||
assert!(b.slice(0..99).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn apply_insert_delete_replace() {
|
||||
let mut b = Buffer::new("hello world");
|
||||
b.apply(&[Edit::replace(0..5, "goodbye")]).unwrap();
|
||||
assert_eq!(b.text(), "goodbye world");
|
||||
b.apply(&[Edit::insert(13, "!")]).unwrap();
|
||||
assert_eq!(b.text(), "goodbye world!");
|
||||
b.apply(&[Edit::delete(7..13)]).unwrap();
|
||||
assert_eq!(b.text(), "goodbye!");
|
||||
b.apply(&[]).unwrap();
|
||||
assert_eq!(b.text(), "goodbye!");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn batch_ranges_refer_to_the_original_text_in_any_order() {
|
||||
let mut b = Buffer::new("abcdef");
|
||||
b.apply(&[
|
||||
Edit::replace(4..6, "EF"),
|
||||
Edit::replace(0..1, "AA"),
|
||||
Edit::delete(2..3),
|
||||
])
|
||||
.unwrap();
|
||||
assert_eq!(b.text(), "AAbdEF");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn insertions_at_one_offset_keep_their_order() {
|
||||
let mut b = Buffer::new("ab");
|
||||
b.apply(&[
|
||||
Edit::insert(1, "1"),
|
||||
Edit::replace(1..2, "B"),
|
||||
Edit::insert(1, "2"),
|
||||
])
|
||||
.unwrap();
|
||||
assert_eq!(b.text(), "a12B");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn overlapping_edits_are_rejected_and_leave_the_buffer_unchanged() {
|
||||
let mut b = Buffer::new("abcdef");
|
||||
assert_eq!(
|
||||
b.apply(&[Edit::delete(3..5), Edit::delete(1..4)]),
|
||||
Err(TextError::OverlappingEdits {
|
||||
first: 1..4,
|
||||
second: 3..5
|
||||
})
|
||||
);
|
||||
assert_eq!(b.text(), "abcdef");
|
||||
assert!(
|
||||
b.apply(&[Edit::delete(1..4), Edit::insert(2, "x")])
|
||||
.is_err()
|
||||
);
|
||||
assert_eq!(b.text(), "abcdef");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn invalid_edits_are_rejected_and_leave_the_buffer_unchanged() {
|
||||
let mut b = Buffer::new("a😀");
|
||||
let bad = [
|
||||
Edit::insert(2, "x"),
|
||||
Edit::delete(0..9),
|
||||
Edit::replace(Range { start: 1, end: 0 }, "x"),
|
||||
Edit::insert(usize::MAX, "x"),
|
||||
];
|
||||
for edit in bad {
|
||||
assert!(b.apply(&[Edit::insert(0, "ok"), edit]).is_err());
|
||||
assert_eq!(b.text(), "a😀");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn edits_may_split_and_join_crlf() {
|
||||
let mut b = Buffer::new("a\r\nb");
|
||||
b.apply(&[Edit::insert(2, "x")]).unwrap();
|
||||
assert_eq!(b.line_count(), 3);
|
||||
b.apply(&[Edit::delete(2..3)]).unwrap();
|
||||
assert_eq!(b.line_count(), 2);
|
||||
assert_eq!(b.line_text(1).unwrap(), "b");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn keeps_the_files_line_ending_on_save() {
|
||||
let mut crlf = Buffer::new("a\r\nb\r\n");
|
||||
assert_eq!(crlf.line_ending(), LineEnding::CrLf);
|
||||
crlf.apply(&[Edit::insert(1, "\nnew")]).unwrap();
|
||||
assert_eq!(crlf.save_text(), "a\r\nnew\r\nb\r\n");
|
||||
|
||||
let mut cr = Buffer::new("a\rb");
|
||||
assert_eq!(cr.line_ending(), LineEnding::Cr);
|
||||
cr.apply(&[Edit::insert(3, "\r\nc")]).unwrap();
|
||||
assert_eq!(cr.save_text(), "a\rb\rc");
|
||||
|
||||
let plain = Buffer::new("no break");
|
||||
assert_eq!(plain.line_ending(), LineEnding::Lf);
|
||||
|
||||
let mut lf = Buffer::new("a\nb");
|
||||
lf.set_line_ending(LineEnding::CrLf);
|
||||
assert_eq!(lf.save_text(), "a\r\nb");
|
||||
assert_eq!(lf.text(), "a\nb");
|
||||
}
|
||||
}
|
||||
36
crates/text/src/edit.rs
Normal file
36
crates/text/src/edit.rs
Normal file
|
|
@ -0,0 +1,36 @@
|
|||
//! Text edits addressed by byte ranges.
|
||||
|
||||
use std::ops::Range;
|
||||
|
||||
/// Replaces the bytes in `range` with `text`.
|
||||
///
|
||||
/// An empty range inserts, an empty `text` deletes. Ranges are byte offsets
|
||||
/// that must lie on UTF-8 character boundaries of the text the edit is
|
||||
/// applied to; see [`crate::Buffer::apply`] for how a batch is interpreted.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Hash, Default)]
|
||||
pub struct Edit {
|
||||
/// Byte range to replace.
|
||||
pub range: Range<usize>,
|
||||
/// Replacement text.
|
||||
pub text: String,
|
||||
}
|
||||
|
||||
impl Edit {
|
||||
/// Replaces `range` with `text`.
|
||||
pub fn replace(range: Range<usize>, text: impl Into<String>) -> Self {
|
||||
Self {
|
||||
range,
|
||||
text: text.into(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Inserts `text` at byte offset `at`.
|
||||
pub fn insert(at: usize, text: impl Into<String>) -> Self {
|
||||
Self::replace(at..at, text)
|
||||
}
|
||||
|
||||
/// Deletes the bytes in `range`.
|
||||
pub fn delete(range: Range<usize>) -> Self {
|
||||
Self::replace(range, String::new())
|
||||
}
|
||||
}
|
||||
151
crates/text/src/error.rs
Normal file
151
crates/text/src/error.rs
Normal file
|
|
@ -0,0 +1,151 @@
|
|||
//! Errors returned by position conversions and edits.
|
||||
|
||||
use std::fmt;
|
||||
use std::ops::Range;
|
||||
|
||||
/// Why a position conversion or an edit was rejected.
|
||||
///
|
||||
/// Every conversion and edit on [`crate::Buffer`] returns this instead of
|
||||
/// panicking, so callers can turn bad offsets from untrusted input (LSP
|
||||
/// messages, frontend edits, command params) into an error message.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub enum TextError {
|
||||
/// A byte offset lies past the end of the text.
|
||||
OffsetOutOfRange {
|
||||
/// The rejected byte offset.
|
||||
offset: usize,
|
||||
/// The text length in bytes.
|
||||
len: usize,
|
||||
},
|
||||
/// A byte offset lies inside a multi-byte UTF-8 character.
|
||||
NotCharBoundary {
|
||||
/// The rejected byte offset.
|
||||
offset: usize,
|
||||
},
|
||||
/// A byte offset lies between the `\r` and the `\n` of a CRLF line
|
||||
/// break, where no line and column exist.
|
||||
InsideLineBreak {
|
||||
/// The rejected byte offset.
|
||||
offset: usize,
|
||||
},
|
||||
/// A UTF-16 offset lies past the end of the text.
|
||||
Utf16OutOfRange {
|
||||
/// The rejected offset in UTF-16 code units.
|
||||
offset: usize,
|
||||
/// The text length in UTF-16 code units.
|
||||
len: usize,
|
||||
},
|
||||
/// A UTF-16 offset lies between the two halves of a surrogate pair.
|
||||
Utf16InsideChar {
|
||||
/// The rejected offset in UTF-16 code units.
|
||||
offset: usize,
|
||||
},
|
||||
/// A line index is not smaller than the line count.
|
||||
LineOutOfRange {
|
||||
/// The rejected zero-based line index.
|
||||
line: usize,
|
||||
/// The number of lines in the text.
|
||||
line_count: usize,
|
||||
},
|
||||
/// A column lies past the end of its line (line break excluded).
|
||||
ColumnOutOfRange {
|
||||
/// The zero-based line index.
|
||||
line: usize,
|
||||
/// The rejected column.
|
||||
column: usize,
|
||||
/// The line length in the column's unit, line break excluded.
|
||||
line_len: usize,
|
||||
},
|
||||
/// A UTF-16 column lies between the two halves of a surrogate pair.
|
||||
ColumnInsideChar {
|
||||
/// The zero-based line index.
|
||||
line: usize,
|
||||
/// The rejected column in UTF-16 code units.
|
||||
column: usize,
|
||||
},
|
||||
/// A range whose start lies after its end.
|
||||
InvalidRange {
|
||||
/// The range start in bytes.
|
||||
start: usize,
|
||||
/// The range end in bytes.
|
||||
end: usize,
|
||||
},
|
||||
/// Two edits of one batch touch overlapping text.
|
||||
OverlappingEdits {
|
||||
/// The byte range of the edit that starts first.
|
||||
first: Range<usize>,
|
||||
/// The byte range of the edit that overlaps it.
|
||||
second: Range<usize>,
|
||||
},
|
||||
/// The rope rejected an operation that validation had accepted; a bug in
|
||||
/// this crate rather than bad input.
|
||||
Internal {
|
||||
/// The rope's own error message.
|
||||
detail: String,
|
||||
},
|
||||
}
|
||||
|
||||
impl fmt::Display for TextError {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
match self {
|
||||
Self::OffsetOutOfRange { offset, len } => {
|
||||
write!(
|
||||
f,
|
||||
"byte offset {offset} is past the end of the text ({len} bytes)"
|
||||
)
|
||||
}
|
||||
Self::NotCharBoundary { offset } => {
|
||||
write!(f, "byte offset {offset} is inside a UTF-8 character")
|
||||
}
|
||||
Self::InsideLineBreak { offset } => {
|
||||
write!(f, "byte offset {offset} is inside a CRLF line break")
|
||||
}
|
||||
Self::Utf16OutOfRange { offset, len } => write!(
|
||||
f,
|
||||
"UTF-16 offset {offset} is past the end of the text ({len} code units)"
|
||||
),
|
||||
Self::Utf16InsideChar { offset } => {
|
||||
write!(f, "UTF-16 offset {offset} is inside a surrogate pair")
|
||||
}
|
||||
Self::LineOutOfRange { line, line_count } => {
|
||||
write!(
|
||||
f,
|
||||
"line {line} does not exist (the text has {line_count} lines)"
|
||||
)
|
||||
}
|
||||
Self::ColumnOutOfRange {
|
||||
line,
|
||||
column,
|
||||
line_len,
|
||||
} => write!(
|
||||
f,
|
||||
"column {column} is past the end of line {line} (length {line_len})"
|
||||
),
|
||||
Self::ColumnInsideChar { line, column } => {
|
||||
write!(
|
||||
f,
|
||||
"column {column} of line {line} is inside a surrogate pair"
|
||||
)
|
||||
}
|
||||
Self::InvalidRange { start, end } => {
|
||||
write!(f, "range start {start} is after its end {end}")
|
||||
}
|
||||
Self::OverlappingEdits { first, second } => write!(
|
||||
f,
|
||||
"edits {}..{} and {}..{} overlap",
|
||||
first.start, first.end, second.start, second.end
|
||||
),
|
||||
Self::Internal { detail } => write!(f, "internal text buffer error: {detail}"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl std::error::Error for TextError {}
|
||||
|
||||
impl From<ropey::Error> for TextError {
|
||||
fn from(error: ropey::Error) -> Self {
|
||||
Self::Internal {
|
||||
detail: error.to_string(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -3,3 +3,22 @@
|
|||
//! Holds the rope buffer, line and column positions, UTF-8/UTF-16 offset
|
||||
//! conversion and edit application. Every other crate converts text offsets
|
||||
//! through this crate instead of slicing strings by hand.
|
||||
//!
|
||||
//! - [`Buffer`]: a [`ropey`] rope plus the file's [`LineEnding`].
|
||||
//! - [`Position`] and [`ColumnUnit`]: zero-based line and column, the
|
||||
//! column in UTF-16 code units (LSP, CodeMirror) or in chars.
|
||||
//! - [`Edit`] and [`Buffer::apply`]: atomic batches of byte-range edits.
|
||||
//! - [`TextError`]: every conversion and edit is fallible instead of
|
||||
//! panicking on bad offsets.
|
||||
|
||||
mod buffer;
|
||||
mod edit;
|
||||
mod error;
|
||||
pub mod line_ending;
|
||||
mod position;
|
||||
|
||||
pub use buffer::Buffer;
|
||||
pub use edit::Edit;
|
||||
pub use error::TextError;
|
||||
pub use line_ending::LineEnding;
|
||||
pub use position::{ColumnUnit, Position};
|
||||
|
|
|
|||
178
crates/text/src/line_ending.rs
Normal file
178
crates/text/src/line_ending.rs
Normal file
|
|
@ -0,0 +1,178 @@
|
|||
//! Line ending detection and normalisation.
|
||||
//!
|
||||
//! Lutra recognises exactly the three line breaks that LSP and CodeMirror
|
||||
//! know: `\n`, `\r\n` and a lone `\r`. Unicode separators such as U+2028 are
|
||||
//! ordinary characters.
|
||||
//!
|
||||
//! Save policy: a buffer remembers the line ending its file was opened with
|
||||
//! ([`LineEnding::detect`], falling back to [`LineEnding::Lf`] for files
|
||||
//! without any line break) and [`crate::Buffer::save_text`] writes every line
|
||||
//! break with that ending. Line breaks typed in the editor therefore take the
|
||||
//! file's ending, and a file keeps its ending across edits. A file with mixed
|
||||
//! endings is saved with its dominant one.
|
||||
|
||||
/// One of the three recognised line breaks.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Default)]
|
||||
pub enum LineEnding {
|
||||
/// `\n`, the Unix ending and the one PSR-12 requires for PHP files.
|
||||
#[default]
|
||||
Lf,
|
||||
/// `\r\n`, the Windows ending.
|
||||
CrLf,
|
||||
/// A lone `\r`, the classic Mac OS ending.
|
||||
Cr,
|
||||
}
|
||||
|
||||
impl LineEnding {
|
||||
/// The line break as text.
|
||||
pub const fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::Lf => "\n",
|
||||
Self::CrLf => "\r\n",
|
||||
Self::Cr => "\r",
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns the most frequent line ending in `text`, or `None` if the
|
||||
/// text has no line break. Ties go to `Lf`, then `CrLf`, then `Cr`.
|
||||
pub fn detect(text: &str) -> Option<Self> {
|
||||
Self::detect_chunks(std::iter::once(text))
|
||||
}
|
||||
|
||||
/// Like [`LineEnding::detect`] for text split into chunks, such as the
|
||||
/// chunks of a rope. A CRLF split across two chunks counts once.
|
||||
pub fn detect_chunks<'a>(chunks: impl IntoIterator<Item = &'a str>) -> Option<Self> {
|
||||
let mut counts = Counts::default();
|
||||
let mut pending_cr = false;
|
||||
for ch in chunks.into_iter().flat_map(str::chars) {
|
||||
if pending_cr {
|
||||
pending_cr = false;
|
||||
if ch == '\n' {
|
||||
counts.crlf += 1;
|
||||
continue;
|
||||
}
|
||||
counts.cr += 1;
|
||||
}
|
||||
match ch {
|
||||
'\r' => pending_cr = true,
|
||||
'\n' => counts.lf += 1,
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
if pending_cr {
|
||||
counts.cr += 1;
|
||||
}
|
||||
counts.dominant()
|
||||
}
|
||||
|
||||
/// Returns `text` with every line break replaced by this ending.
|
||||
pub fn normalize(self, text: &str) -> String {
|
||||
self.normalize_chunks(std::iter::once(text))
|
||||
}
|
||||
|
||||
/// Like [`LineEnding::normalize`] for text split into chunks. A CRLF
|
||||
/// split across two chunks becomes a single line break.
|
||||
pub fn normalize_chunks<'a>(self, chunks: impl IntoIterator<Item = &'a str>) -> String {
|
||||
let ending = self.as_str();
|
||||
let mut out = String::new();
|
||||
let mut pending_cr = false;
|
||||
for chunk in chunks {
|
||||
out.reserve(chunk.len());
|
||||
for ch in chunk.chars() {
|
||||
if pending_cr {
|
||||
pending_cr = false;
|
||||
out.push_str(ending);
|
||||
if ch == '\n' {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
match ch {
|
||||
'\r' => pending_cr = true,
|
||||
'\n' => out.push_str(ending),
|
||||
other => out.push(other),
|
||||
}
|
||||
}
|
||||
}
|
||||
if pending_cr {
|
||||
out.push_str(ending);
|
||||
}
|
||||
out
|
||||
}
|
||||
}
|
||||
|
||||
/// Number of line breaks of each kind.
|
||||
#[derive(Default)]
|
||||
struct Counts {
|
||||
/// Count of `\n` not preceded by `\r`.
|
||||
lf: usize,
|
||||
/// Count of `\r\n`.
|
||||
crlf: usize,
|
||||
/// Count of `\r` not followed by `\n`.
|
||||
cr: usize,
|
||||
}
|
||||
|
||||
impl Counts {
|
||||
/// The most frequent ending, ties broken in declaration order.
|
||||
fn dominant(&self) -> Option<LineEnding> {
|
||||
let mut best: Option<(LineEnding, usize)> = None;
|
||||
for (ending, count) in [
|
||||
(LineEnding::Lf, self.lf),
|
||||
(LineEnding::CrLf, self.crlf),
|
||||
(LineEnding::Cr, self.cr),
|
||||
] {
|
||||
if count > best.map_or(0, |(_, n)| n) {
|
||||
best = Some((ending, count));
|
||||
}
|
||||
}
|
||||
best.map(|(ending, _)| ending)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn detects_each_ending() {
|
||||
assert_eq!(LineEnding::detect("a\nb"), Some(LineEnding::Lf));
|
||||
assert_eq!(LineEnding::detect("a\r\nb"), Some(LineEnding::CrLf));
|
||||
assert_eq!(LineEnding::detect("a\rb"), Some(LineEnding::Cr));
|
||||
assert_eq!(LineEnding::detect("a\r"), Some(LineEnding::Cr));
|
||||
assert_eq!(LineEnding::detect("no break"), None);
|
||||
assert_eq!(LineEnding::detect(""), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn detect_takes_the_majority_and_breaks_ties_in_order() {
|
||||
assert_eq!(LineEnding::detect("a\r\nb\r\nc\n"), Some(LineEnding::CrLf));
|
||||
assert_eq!(LineEnding::detect("a\r\nb\n"), Some(LineEnding::Lf));
|
||||
assert_eq!(LineEnding::detect("a\rb\r\n"), Some(LineEnding::CrLf));
|
||||
assert_eq!(LineEnding::detect("\r\r\n\n"), Some(LineEnding::Lf));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unicode_separators_are_not_line_breaks() {
|
||||
assert_eq!(LineEnding::detect("a\u{2028}b\u{85}c"), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn crlf_split_across_chunks_counts_once() {
|
||||
assert_eq!(
|
||||
LineEnding::detect_chunks(["a\r", "\nb\r", "\n"]),
|
||||
Some(LineEnding::CrLf)
|
||||
);
|
||||
assert_eq!(
|
||||
LineEnding::Lf.normalize_chunks(["a\r", "\nb"]),
|
||||
"a\nb".to_string()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn normalize_rewrites_every_break() {
|
||||
let mixed = "a\nb\r\nc\rd\r";
|
||||
assert_eq!(LineEnding::Lf.normalize(mixed), "a\nb\nc\nd\n");
|
||||
assert_eq!(LineEnding::CrLf.normalize(mixed), "a\r\nb\r\nc\r\nd\r\n");
|
||||
assert_eq!(LineEnding::Cr.normalize(mixed), "a\rb\rc\rd\r");
|
||||
assert_eq!(LineEnding::CrLf.normalize("\r\r\n"), "\r\n\r\n");
|
||||
}
|
||||
}
|
||||
39
crates/text/src/position.rs
Normal file
39
crates/text/src/position.rs
Normal file
|
|
@ -0,0 +1,39 @@
|
|||
//! Line and column positions.
|
||||
//!
|
||||
//! LSP counts columns in UTF-16 code units by default
|
||||
//! (<https://microsoft.github.io/language-server-protocol/specifications/lsp/3.17/specification/#textDocuments>)
|
||||
//! and CodeMirror addresses text in UTF-16 code units
|
||||
//! (<https://codemirror.net/docs/ref/#state.Text>), so UTF-16 is the unit
|
||||
//! that crosses the frontend and LSP boundaries. Chars (Unicode scalar
|
||||
//! values) are offered for Rust-side code that walks text by `char`.
|
||||
|
||||
/// The unit a column is counted in.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
|
||||
pub enum ColumnUnit {
|
||||
/// UTF-16 code units: characters outside the Basic Multilingual Plane,
|
||||
/// such as most emoji, count two.
|
||||
Utf16,
|
||||
/// Unicode scalar values (Rust `char`s): every code point counts one,
|
||||
/// including each combining mark.
|
||||
Char,
|
||||
}
|
||||
|
||||
/// A zero-based line and column.
|
||||
///
|
||||
/// The column's unit is not stored; conversions take a [`ColumnUnit`]. A
|
||||
/// column never reaches into the line break: the largest valid column is
|
||||
/// the line's length without its line break.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Default)]
|
||||
pub struct Position {
|
||||
/// Zero-based line index.
|
||||
pub line: usize,
|
||||
/// Zero-based column in the unit given to the conversion.
|
||||
pub column: usize,
|
||||
}
|
||||
|
||||
impl Position {
|
||||
/// Creates a position from a zero-based line and column.
|
||||
pub const fn new(line: usize, column: usize) -> Self {
|
||||
Self { line, column }
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue