M2.1: tree-sitter PHP in crates/syntax

Parser (parse, incremental reparse from lutra_text edits), outline with
UTF-16 ranges and short signatures, folding ranges, 8 MiB parse cap.
Snapshot tests for five PHP 8.3 fixtures, proptests (random bytes,
reparse equals fresh parse, undo round trip), ignored release timing
test. Docs: architecture Syntax section, parity evidence, G4 note.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
danny 2026-10-16 22:06:21 +02:00
parent d12a517009
commit a5236acd7e
25 changed files with 1768 additions and 6 deletions

273
crates/syntax/src/parser.rs Normal file
View file

@ -0,0 +1,273 @@
//! The PHP parser and its syntax tree.
//!
//! [`Parser`] wraps a tree-sitter parser loaded with the PHP grammar
//! (`tree-sitter-php`, the variant that accepts inline HTML around
//! `<?php` tags, as `.php` files do). [`Tree`] keeps the text it was parsed
//! from, so an incremental [`Parser::reparse`] needs only the edits.
use std::fmt;
use lutra_text::{Buffer, Edit, TextError};
use tree_sitter::{InputEdit, Node, Point};
/// Files larger than this many bytes (8 MiB) are not parsed.
///
/// A [`Tree`] for such a text reports [`Tree::truncated`], has no syntax
/// nodes and yields an empty outline and no folding ranges.
pub const MAX_PARSE_BYTES: usize = 8 * 1024 * 1024;
/// Errors of the syntax layer.
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum SyntaxError {
/// The PHP grammar could not be loaded into tree-sitter, for example
/// because its ABI version is not supported by the linked library.
Language(String),
/// An edit passed to [`Parser::reparse`] was rejected by `lutra_text`
/// (reversed, out of range, not on a character boundary, overlapping).
Edit(TextError),
}
impl fmt::Display for SyntaxError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::Language(reason) => write!(f, "cannot load the PHP grammar: {reason}"),
Self::Edit(error) => write!(f, "invalid edit: {error}"),
}
}
}
impl std::error::Error for SyntaxError {}
impl From<TextError> for SyntaxError {
fn from(error: TextError) -> Self {
Self::Edit(error)
}
}
/// A parsed PHP text.
///
/// Holds the tree-sitter tree together with the text it was parsed from.
/// Cloning is cheap: the text is a rope and tree-sitter trees are
/// reference counted. Broken PHP still produces a tree; tree-sitter marks
/// the broken parts as error nodes, which the outline and folding skip.
#[derive(Debug, Clone)]
pub struct Tree {
/// The syntax tree; `None` when the text was too large to parse or
/// tree-sitter gave up.
inner: Option<tree_sitter::Tree>,
/// The text the tree describes.
text: Buffer,
/// Whether parsing was skipped because the text exceeds
/// [`MAX_PARSE_BYTES`].
truncated: bool,
}
impl Tree {
/// The text this tree was parsed from.
pub fn text(&self) -> &Buffer {
&self.text
}
/// Whether the text was too large to parse (over [`MAX_PARSE_BYTES`]).
///
/// The engine reports this as `truncated` next to the outline.
pub fn truncated(&self) -> bool {
self.truncated
}
/// Whether the tree contains syntax errors or missing tokens.
pub fn has_errors(&self) -> bool {
self.root().is_some_and(|root| root.has_error())
}
/// The tree in tree-sitter's S-expression notation, without positions.
///
/// Meant for tests and diagnostics; empty when nothing was parsed.
pub fn to_sexp(&self) -> String {
self.root().map(|root| root.to_sexp()).unwrap_or_default()
}
/// The root node, if the text was parsed.
pub(crate) fn root(&self) -> Option<Node<'_>> {
self.inner.as_ref().map(tree_sitter::Tree::root_node)
}
}
/// A PHP parser.
///
/// Reuse one parser for many parses; it is cheap to keep but not to
/// create. A parser is `Send` but parses one text at a time (`&mut self`).
pub struct Parser {
/// The tree-sitter parser with the PHP grammar loaded.
inner: tree_sitter::Parser,
}
impl fmt::Debug for Parser {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("Parser").finish_non_exhaustive()
}
}
impl Parser {
/// Creates a parser for PHP files (inline HTML allowed around
/// `<?php … ?>`).
///
/// # Errors
/// Fails with [`SyntaxError::Language`] when the bundled grammar cannot
/// be loaded, which only happens with a mismatched tree-sitter build.
pub fn new() -> Result<Self, SyntaxError> {
let mut inner = tree_sitter::Parser::new();
inner
.set_language(&tree_sitter_php::LANGUAGE_PHP.into())
.map_err(|error| SyntaxError::Language(error.to_string()))?;
Ok(Self { inner })
}
/// Parses `text` from scratch.
///
/// Never fails: broken PHP yields a tree with error nodes, and a text
/// over [`MAX_PARSE_BYTES`] yields an empty, [`Tree::truncated`] tree.
pub fn parse(&mut self, text: &str) -> Tree {
self.parse_buffer(Buffer::new(text), text, None)
}
/// Applies `edits` to the text of `tree` and parses the result
/// incrementally, reusing the unchanged parts of `tree`.
///
/// `edits` follow [`Buffer::apply`]: every range refers to the text of
/// `tree` (before any of the edits), in any order. The batch is mapped
/// to one tree-sitter edit that covers all changed bytes, with
/// positions computed from the old and new text. The result equals a
/// fresh [`Parser::parse`] of the edited text.
///
/// # Errors
/// Fails with [`SyntaxError::Edit`] when `lutra_text` rejects the
/// batch; `tree` is unaffected.
pub fn reparse(&mut self, tree: &Tree, edits: &[Edit]) -> Result<Tree, SyntaxError> {
let mut buffer = tree.text.clone();
buffer.apply(edits)?;
let new_text = buffer.text();
let (Some(old_tree), Some(input_edit)) = (
tree.inner.as_ref(),
covering_edit(&tree.text.text(), &new_text, edits),
) else {
return Ok(self.parse_buffer(buffer, &new_text, None));
};
let mut edited = old_tree.clone();
edited.edit(&input_edit);
Ok(self.parse_buffer(buffer, &new_text, Some(&edited)))
}
/// Parses `text` (the contents of `buffer`), reusing `old` when given.
fn parse_buffer(
&mut self,
buffer: Buffer,
text: &str,
old: Option<&tree_sitter::Tree>,
) -> Tree {
if text.len() > MAX_PARSE_BYTES {
return Tree {
inner: None,
text: buffer,
truncated: true,
};
}
Tree {
inner: self.inner.parse(text, old),
text: buffer,
truncated: false,
}
}
}
/// Builds the single tree-sitter edit that turns `old` into `new` for a
/// batch of `edits` that `lutra_text` already accepted against `old`.
///
/// The edit spans from the smallest start to the largest end of the batch,
/// which keeps it correct however the batch orders insertions that share
/// an offset. Returns `None` for an empty batch or inconsistent lengths, in
/// which case the caller parses from scratch.
fn covering_edit(old: &str, new: &str, edits: &[Edit]) -> Option<InputEdit> {
let start_byte = edits.iter().map(|edit| edit.range.start).min()?;
let old_end_byte = edits.iter().map(|edit| edit.range.end).max()?;
let new_end_byte = old_end_byte
.checked_add(new.len())?
.checked_sub(old.len())?;
if new_end_byte < start_byte {
return None;
}
Some(InputEdit {
start_byte,
old_end_byte,
new_end_byte,
start_position: point_at(old, start_byte),
old_end_position: point_at(old, old_end_byte),
new_end_position: point_at(new, new_end_byte),
})
}
/// The tree-sitter point (row, byte column) of byte `offset` in `text`.
///
/// tree-sitter counts rows by `\n` only. An offset past the end is clamped
/// to the end of the text.
fn point_at(text: &str, offset: usize) -> Point {
let bytes = text.as_bytes();
let prefix = bytes.get(..offset).unwrap_or(bytes);
let row = prefix.iter().filter(|byte| **byte == b'\n').count();
let line_start = prefix
.iter()
.rposition(|byte| *byte == b'\n')
.map_or(0, |newline| newline.saturating_add(1));
Point {
row,
column: prefix.len().saturating_sub(line_start),
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn point_counts_rows_and_byte_columns() {
let text = "ab\ncä\n";
assert_eq!(point_at(text, 0), Point { row: 0, column: 0 });
assert_eq!(point_at(text, 3), Point { row: 1, column: 0 });
assert_eq!(point_at(text, 6), Point { row: 1, column: 3 });
assert_eq!(point_at(text, 7), Point { row: 2, column: 0 });
assert_eq!(point_at(text, 99), Point { row: 2, column: 0 });
}
#[test]
fn oversized_text_is_not_parsed() {
let mut parser = Parser::new().unwrap();
let text = format!("<?php\n/*{}*/", "x".repeat(MAX_PARSE_BYTES));
let tree = parser.parse(&text);
assert!(tree.truncated());
assert!(tree.to_sexp().is_empty());
let shrunk = parser
.reparse(&tree, &[Edit::delete(8..MAX_PARSE_BYTES)])
.unwrap();
assert!(!shrunk.truncated());
assert_eq!(
shrunk.to_sexp(),
parser.parse("<?php\n/*xxxxxxxx*/").to_sexp()
);
}
#[test]
fn rejected_edit_is_an_error() {
let mut parser = Parser::new().unwrap();
let tree = parser.parse("<?php echo 1;");
let error = parser.reparse(&tree, &[Edit::delete(5..99)]).unwrap_err();
assert!(matches!(error, SyntaxError::Edit(_)));
}
#[test]
fn empty_batch_reparses_to_the_same_tree() {
let mut parser = Parser::new().unwrap();
let tree = parser.parse("<?php function f() {}");
let again = parser.reparse(&tree, &[]).unwrap();
assert_eq!(again.to_sexp(), tree.to_sexp());
}
}