Parser (parse, incremental reparse from lutra_text edits), outline with UTF-16 ranges and short signatures, folding ranges, 8 MiB parse cap. Snapshot tests for five PHP 8.3 fixtures, proptests (random bytes, reparse equals fresh parse, undo round trip), ignored release timing test. Docs: architecture Syntax section, parity evidence, G4 note. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
273 lines
9.4 KiB
Rust
273 lines
9.4 KiB
Rust
//! The PHP parser and its syntax tree.
|
|
//!
|
|
//! [`Parser`] wraps a tree-sitter parser loaded with the PHP grammar
|
|
//! (`tree-sitter-php`, the variant that accepts inline HTML around
|
|
//! `<?php` tags, as `.php` files do). [`Tree`] keeps the text it was parsed
|
|
//! from, so an incremental [`Parser::reparse`] needs only the edits.
|
|
|
|
use std::fmt;
|
|
|
|
use lutra_text::{Buffer, Edit, TextError};
|
|
use tree_sitter::{InputEdit, Node, Point};
|
|
|
|
/// Files larger than this many bytes (8 MiB) are not parsed.
|
|
///
|
|
/// A [`Tree`] for such a text reports [`Tree::truncated`], has no syntax
|
|
/// nodes and yields an empty outline and no folding ranges.
|
|
pub const MAX_PARSE_BYTES: usize = 8 * 1024 * 1024;
|
|
|
|
/// Errors of the syntax layer.
|
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
|
pub enum SyntaxError {
|
|
/// The PHP grammar could not be loaded into tree-sitter, for example
|
|
/// because its ABI version is not supported by the linked library.
|
|
Language(String),
|
|
/// An edit passed to [`Parser::reparse`] was rejected by `lutra_text`
|
|
/// (reversed, out of range, not on a character boundary, overlapping).
|
|
Edit(TextError),
|
|
}
|
|
|
|
impl fmt::Display for SyntaxError {
|
|
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
|
match self {
|
|
Self::Language(reason) => write!(f, "cannot load the PHP grammar: {reason}"),
|
|
Self::Edit(error) => write!(f, "invalid edit: {error}"),
|
|
}
|
|
}
|
|
}
|
|
|
|
impl std::error::Error for SyntaxError {}
|
|
|
|
impl From<TextError> for SyntaxError {
|
|
fn from(error: TextError) -> Self {
|
|
Self::Edit(error)
|
|
}
|
|
}
|
|
|
|
/// A parsed PHP text.
|
|
///
|
|
/// Holds the tree-sitter tree together with the text it was parsed from.
|
|
/// Cloning is cheap: the text is a rope and tree-sitter trees are
|
|
/// reference counted. Broken PHP still produces a tree; tree-sitter marks
|
|
/// the broken parts as error nodes, which the outline and folding skip.
|
|
#[derive(Debug, Clone)]
|
|
pub struct Tree {
|
|
/// The syntax tree; `None` when the text was too large to parse or
|
|
/// tree-sitter gave up.
|
|
inner: Option<tree_sitter::Tree>,
|
|
/// The text the tree describes.
|
|
text: Buffer,
|
|
/// Whether parsing was skipped because the text exceeds
|
|
/// [`MAX_PARSE_BYTES`].
|
|
truncated: bool,
|
|
}
|
|
|
|
impl Tree {
|
|
/// The text this tree was parsed from.
|
|
pub fn text(&self) -> &Buffer {
|
|
&self.text
|
|
}
|
|
|
|
/// Whether the text was too large to parse (over [`MAX_PARSE_BYTES`]).
|
|
///
|
|
/// The engine reports this as `truncated` next to the outline.
|
|
pub fn truncated(&self) -> bool {
|
|
self.truncated
|
|
}
|
|
|
|
/// Whether the tree contains syntax errors or missing tokens.
|
|
pub fn has_errors(&self) -> bool {
|
|
self.root().is_some_and(|root| root.has_error())
|
|
}
|
|
|
|
/// The tree in tree-sitter's S-expression notation, without positions.
|
|
///
|
|
/// Meant for tests and diagnostics; empty when nothing was parsed.
|
|
pub fn to_sexp(&self) -> String {
|
|
self.root().map(|root| root.to_sexp()).unwrap_or_default()
|
|
}
|
|
|
|
/// The root node, if the text was parsed.
|
|
pub(crate) fn root(&self) -> Option<Node<'_>> {
|
|
self.inner.as_ref().map(tree_sitter::Tree::root_node)
|
|
}
|
|
}
|
|
|
|
/// A PHP parser.
|
|
///
|
|
/// Reuse one parser for many parses; it is cheap to keep but not to
|
|
/// create. A parser is `Send` but parses one text at a time (`&mut self`).
|
|
pub struct Parser {
|
|
/// The tree-sitter parser with the PHP grammar loaded.
|
|
inner: tree_sitter::Parser,
|
|
}
|
|
|
|
impl fmt::Debug for Parser {
|
|
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
|
f.debug_struct("Parser").finish_non_exhaustive()
|
|
}
|
|
}
|
|
|
|
impl Parser {
|
|
/// Creates a parser for PHP files (inline HTML allowed around
|
|
/// `<?php … ?>`).
|
|
///
|
|
/// # Errors
|
|
/// Fails with [`SyntaxError::Language`] when the bundled grammar cannot
|
|
/// be loaded, which only happens with a mismatched tree-sitter build.
|
|
pub fn new() -> Result<Self, SyntaxError> {
|
|
let mut inner = tree_sitter::Parser::new();
|
|
inner
|
|
.set_language(&tree_sitter_php::LANGUAGE_PHP.into())
|
|
.map_err(|error| SyntaxError::Language(error.to_string()))?;
|
|
Ok(Self { inner })
|
|
}
|
|
|
|
/// Parses `text` from scratch.
|
|
///
|
|
/// Never fails: broken PHP yields a tree with error nodes, and a text
|
|
/// over [`MAX_PARSE_BYTES`] yields an empty, [`Tree::truncated`] tree.
|
|
pub fn parse(&mut self, text: &str) -> Tree {
|
|
self.parse_buffer(Buffer::new(text), text, None)
|
|
}
|
|
|
|
/// Applies `edits` to the text of `tree` and parses the result
|
|
/// incrementally, reusing the unchanged parts of `tree`.
|
|
///
|
|
/// `edits` follow [`Buffer::apply`]: every range refers to the text of
|
|
/// `tree` (before any of the edits), in any order. The batch is mapped
|
|
/// to one tree-sitter edit that covers all changed bytes, with
|
|
/// positions computed from the old and new text. The result equals a
|
|
/// fresh [`Parser::parse`] of the edited text.
|
|
///
|
|
/// # Errors
|
|
/// Fails with [`SyntaxError::Edit`] when `lutra_text` rejects the
|
|
/// batch; `tree` is unaffected.
|
|
pub fn reparse(&mut self, tree: &Tree, edits: &[Edit]) -> Result<Tree, SyntaxError> {
|
|
let mut buffer = tree.text.clone();
|
|
buffer.apply(edits)?;
|
|
let new_text = buffer.text();
|
|
let (Some(old_tree), Some(input_edit)) = (
|
|
tree.inner.as_ref(),
|
|
covering_edit(&tree.text.text(), &new_text, edits),
|
|
) else {
|
|
return Ok(self.parse_buffer(buffer, &new_text, None));
|
|
};
|
|
let mut edited = old_tree.clone();
|
|
edited.edit(&input_edit);
|
|
Ok(self.parse_buffer(buffer, &new_text, Some(&edited)))
|
|
}
|
|
|
|
/// Parses `text` (the contents of `buffer`), reusing `old` when given.
|
|
fn parse_buffer(
|
|
&mut self,
|
|
buffer: Buffer,
|
|
text: &str,
|
|
old: Option<&tree_sitter::Tree>,
|
|
) -> Tree {
|
|
if text.len() > MAX_PARSE_BYTES {
|
|
return Tree {
|
|
inner: None,
|
|
text: buffer,
|
|
truncated: true,
|
|
};
|
|
}
|
|
Tree {
|
|
inner: self.inner.parse(text, old),
|
|
text: buffer,
|
|
truncated: false,
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Builds the single tree-sitter edit that turns `old` into `new` for a
|
|
/// batch of `edits` that `lutra_text` already accepted against `old`.
|
|
///
|
|
/// The edit spans from the smallest start to the largest end of the batch,
|
|
/// which keeps it correct however the batch orders insertions that share
|
|
/// an offset. Returns `None` for an empty batch or inconsistent lengths, in
|
|
/// which case the caller parses from scratch.
|
|
fn covering_edit(old: &str, new: &str, edits: &[Edit]) -> Option<InputEdit> {
|
|
let start_byte = edits.iter().map(|edit| edit.range.start).min()?;
|
|
let old_end_byte = edits.iter().map(|edit| edit.range.end).max()?;
|
|
let new_end_byte = old_end_byte
|
|
.checked_add(new.len())?
|
|
.checked_sub(old.len())?;
|
|
if new_end_byte < start_byte {
|
|
return None;
|
|
}
|
|
Some(InputEdit {
|
|
start_byte,
|
|
old_end_byte,
|
|
new_end_byte,
|
|
start_position: point_at(old, start_byte),
|
|
old_end_position: point_at(old, old_end_byte),
|
|
new_end_position: point_at(new, new_end_byte),
|
|
})
|
|
}
|
|
|
|
/// The tree-sitter point (row, byte column) of byte `offset` in `text`.
|
|
///
|
|
/// tree-sitter counts rows by `\n` only. An offset past the end is clamped
|
|
/// to the end of the text.
|
|
fn point_at(text: &str, offset: usize) -> Point {
|
|
let bytes = text.as_bytes();
|
|
let prefix = bytes.get(..offset).unwrap_or(bytes);
|
|
let row = prefix.iter().filter(|byte| **byte == b'\n').count();
|
|
let line_start = prefix
|
|
.iter()
|
|
.rposition(|byte| *byte == b'\n')
|
|
.map_or(0, |newline| newline.saturating_add(1));
|
|
Point {
|
|
row,
|
|
column: prefix.len().saturating_sub(line_start),
|
|
}
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
|
|
#[test]
|
|
fn point_counts_rows_and_byte_columns() {
|
|
let text = "ab\ncä\n";
|
|
assert_eq!(point_at(text, 0), Point { row: 0, column: 0 });
|
|
assert_eq!(point_at(text, 3), Point { row: 1, column: 0 });
|
|
assert_eq!(point_at(text, 6), Point { row: 1, column: 3 });
|
|
assert_eq!(point_at(text, 7), Point { row: 2, column: 0 });
|
|
assert_eq!(point_at(text, 99), Point { row: 2, column: 0 });
|
|
}
|
|
|
|
#[test]
|
|
fn oversized_text_is_not_parsed() {
|
|
let mut parser = Parser::new().unwrap();
|
|
let text = format!("<?php\n/*{}*/", "x".repeat(MAX_PARSE_BYTES));
|
|
let tree = parser.parse(&text);
|
|
assert!(tree.truncated());
|
|
assert!(tree.to_sexp().is_empty());
|
|
let shrunk = parser
|
|
.reparse(&tree, &[Edit::delete(8..MAX_PARSE_BYTES)])
|
|
.unwrap();
|
|
assert!(!shrunk.truncated());
|
|
assert_eq!(
|
|
shrunk.to_sexp(),
|
|
parser.parse("<?php\n/*xxxxxxxx*/").to_sexp()
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn rejected_edit_is_an_error() {
|
|
let mut parser = Parser::new().unwrap();
|
|
let tree = parser.parse("<?php echo 1;");
|
|
let error = parser.reparse(&tree, &[Edit::delete(5..99)]).unwrap_err();
|
|
assert!(matches!(error, SyntaxError::Edit(_)));
|
|
}
|
|
|
|
#[test]
|
|
fn empty_batch_reparses_to_the_same_tree() {
|
|
let mut parser = Parser::new().unwrap();
|
|
let tree = parser.parse("<?php function f() {}");
|
|
let again = parser.reparse(&tree, &[]).unwrap();
|
|
assert_eq!(again.to_sexp(), tree.to_sexp());
|
|
}
|
|
}
|