M2.1: tree-sitter PHP in crates/syntax
Parser (parse, incremental reparse from lutra_text edits), outline with UTF-16 ranges and short signatures, folding ranges, 8 MiB parse cap. Snapshot tests for five PHP 8.3 fixtures, proptests (random bytes, reparse equals fresh parse, undo round trip), ignored release timing test. Docs: architecture Syntax section, parity evidence, G4 note. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
d12a517009
commit
a5236acd7e
25 changed files with 1768 additions and 6 deletions
273
crates/syntax/src/parser.rs
Normal file
273
crates/syntax/src/parser.rs
Normal file
|
|
@ -0,0 +1,273 @@
|
|||
//! The PHP parser and its syntax tree.
|
||||
//!
|
||||
//! [`Parser`] wraps a tree-sitter parser loaded with the PHP grammar
|
||||
//! (`tree-sitter-php`, the variant that accepts inline HTML around
|
||||
//! `<?php` tags, as `.php` files do). [`Tree`] keeps the text it was parsed
|
||||
//! from, so an incremental [`Parser::reparse`] needs only the edits.
|
||||
|
||||
use std::fmt;
|
||||
|
||||
use lutra_text::{Buffer, Edit, TextError};
|
||||
use tree_sitter::{InputEdit, Node, Point};
|
||||
|
||||
/// Files larger than this many bytes (8 MiB) are not parsed.
|
||||
///
|
||||
/// A [`Tree`] for such a text reports [`Tree::truncated`], has no syntax
|
||||
/// nodes and yields an empty outline and no folding ranges.
|
||||
pub const MAX_PARSE_BYTES: usize = 8 * 1024 * 1024;
|
||||
|
||||
/// Errors of the syntax layer.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub enum SyntaxError {
|
||||
/// The PHP grammar could not be loaded into tree-sitter, for example
|
||||
/// because its ABI version is not supported by the linked library.
|
||||
Language(String),
|
||||
/// An edit passed to [`Parser::reparse`] was rejected by `lutra_text`
|
||||
/// (reversed, out of range, not on a character boundary, overlapping).
|
||||
Edit(TextError),
|
||||
}
|
||||
|
||||
impl fmt::Display for SyntaxError {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
match self {
|
||||
Self::Language(reason) => write!(f, "cannot load the PHP grammar: {reason}"),
|
||||
Self::Edit(error) => write!(f, "invalid edit: {error}"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl std::error::Error for SyntaxError {}
|
||||
|
||||
impl From<TextError> for SyntaxError {
|
||||
fn from(error: TextError) -> Self {
|
||||
Self::Edit(error)
|
||||
}
|
||||
}
|
||||
|
||||
/// A parsed PHP text.
|
||||
///
|
||||
/// Holds the tree-sitter tree together with the text it was parsed from.
|
||||
/// Cloning is cheap: the text is a rope and tree-sitter trees are
|
||||
/// reference counted. Broken PHP still produces a tree; tree-sitter marks
|
||||
/// the broken parts as error nodes, which the outline and folding skip.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct Tree {
|
||||
/// The syntax tree; `None` when the text was too large to parse or
|
||||
/// tree-sitter gave up.
|
||||
inner: Option<tree_sitter::Tree>,
|
||||
/// The text the tree describes.
|
||||
text: Buffer,
|
||||
/// Whether parsing was skipped because the text exceeds
|
||||
/// [`MAX_PARSE_BYTES`].
|
||||
truncated: bool,
|
||||
}
|
||||
|
||||
impl Tree {
|
||||
/// The text this tree was parsed from.
|
||||
pub fn text(&self) -> &Buffer {
|
||||
&self.text
|
||||
}
|
||||
|
||||
/// Whether the text was too large to parse (over [`MAX_PARSE_BYTES`]).
|
||||
///
|
||||
/// The engine reports this as `truncated` next to the outline.
|
||||
pub fn truncated(&self) -> bool {
|
||||
self.truncated
|
||||
}
|
||||
|
||||
/// Whether the tree contains syntax errors or missing tokens.
|
||||
pub fn has_errors(&self) -> bool {
|
||||
self.root().is_some_and(|root| root.has_error())
|
||||
}
|
||||
|
||||
/// The tree in tree-sitter's S-expression notation, without positions.
|
||||
///
|
||||
/// Meant for tests and diagnostics; empty when nothing was parsed.
|
||||
pub fn to_sexp(&self) -> String {
|
||||
self.root().map(|root| root.to_sexp()).unwrap_or_default()
|
||||
}
|
||||
|
||||
/// The root node, if the text was parsed.
|
||||
pub(crate) fn root(&self) -> Option<Node<'_>> {
|
||||
self.inner.as_ref().map(tree_sitter::Tree::root_node)
|
||||
}
|
||||
}
|
||||
|
||||
/// A PHP parser.
|
||||
///
|
||||
/// Reuse one parser for many parses; it is cheap to keep but not to
|
||||
/// create. A parser is `Send` but parses one text at a time (`&mut self`).
|
||||
pub struct Parser {
|
||||
/// The tree-sitter parser with the PHP grammar loaded.
|
||||
inner: tree_sitter::Parser,
|
||||
}
|
||||
|
||||
impl fmt::Debug for Parser {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
f.debug_struct("Parser").finish_non_exhaustive()
|
||||
}
|
||||
}
|
||||
|
||||
impl Parser {
|
||||
/// Creates a parser for PHP files (inline HTML allowed around
|
||||
/// `<?php … ?>`).
|
||||
///
|
||||
/// # Errors
|
||||
/// Fails with [`SyntaxError::Language`] when the bundled grammar cannot
|
||||
/// be loaded, which only happens with a mismatched tree-sitter build.
|
||||
pub fn new() -> Result<Self, SyntaxError> {
|
||||
let mut inner = tree_sitter::Parser::new();
|
||||
inner
|
||||
.set_language(&tree_sitter_php::LANGUAGE_PHP.into())
|
||||
.map_err(|error| SyntaxError::Language(error.to_string()))?;
|
||||
Ok(Self { inner })
|
||||
}
|
||||
|
||||
/// Parses `text` from scratch.
|
||||
///
|
||||
/// Never fails: broken PHP yields a tree with error nodes, and a text
|
||||
/// over [`MAX_PARSE_BYTES`] yields an empty, [`Tree::truncated`] tree.
|
||||
pub fn parse(&mut self, text: &str) -> Tree {
|
||||
self.parse_buffer(Buffer::new(text), text, None)
|
||||
}
|
||||
|
||||
/// Applies `edits` to the text of `tree` and parses the result
|
||||
/// incrementally, reusing the unchanged parts of `tree`.
|
||||
///
|
||||
/// `edits` follow [`Buffer::apply`]: every range refers to the text of
|
||||
/// `tree` (before any of the edits), in any order. The batch is mapped
|
||||
/// to one tree-sitter edit that covers all changed bytes, with
|
||||
/// positions computed from the old and new text. The result equals a
|
||||
/// fresh [`Parser::parse`] of the edited text.
|
||||
///
|
||||
/// # Errors
|
||||
/// Fails with [`SyntaxError::Edit`] when `lutra_text` rejects the
|
||||
/// batch; `tree` is unaffected.
|
||||
pub fn reparse(&mut self, tree: &Tree, edits: &[Edit]) -> Result<Tree, SyntaxError> {
|
||||
let mut buffer = tree.text.clone();
|
||||
buffer.apply(edits)?;
|
||||
let new_text = buffer.text();
|
||||
let (Some(old_tree), Some(input_edit)) = (
|
||||
tree.inner.as_ref(),
|
||||
covering_edit(&tree.text.text(), &new_text, edits),
|
||||
) else {
|
||||
return Ok(self.parse_buffer(buffer, &new_text, None));
|
||||
};
|
||||
let mut edited = old_tree.clone();
|
||||
edited.edit(&input_edit);
|
||||
Ok(self.parse_buffer(buffer, &new_text, Some(&edited)))
|
||||
}
|
||||
|
||||
/// Parses `text` (the contents of `buffer`), reusing `old` when given.
|
||||
fn parse_buffer(
|
||||
&mut self,
|
||||
buffer: Buffer,
|
||||
text: &str,
|
||||
old: Option<&tree_sitter::Tree>,
|
||||
) -> Tree {
|
||||
if text.len() > MAX_PARSE_BYTES {
|
||||
return Tree {
|
||||
inner: None,
|
||||
text: buffer,
|
||||
truncated: true,
|
||||
};
|
||||
}
|
||||
Tree {
|
||||
inner: self.inner.parse(text, old),
|
||||
text: buffer,
|
||||
truncated: false,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Builds the single tree-sitter edit that turns `old` into `new` for a
|
||||
/// batch of `edits` that `lutra_text` already accepted against `old`.
|
||||
///
|
||||
/// The edit spans from the smallest start to the largest end of the batch,
|
||||
/// which keeps it correct however the batch orders insertions that share
|
||||
/// an offset. Returns `None` for an empty batch or inconsistent lengths, in
|
||||
/// which case the caller parses from scratch.
|
||||
fn covering_edit(old: &str, new: &str, edits: &[Edit]) -> Option<InputEdit> {
|
||||
let start_byte = edits.iter().map(|edit| edit.range.start).min()?;
|
||||
let old_end_byte = edits.iter().map(|edit| edit.range.end).max()?;
|
||||
let new_end_byte = old_end_byte
|
||||
.checked_add(new.len())?
|
||||
.checked_sub(old.len())?;
|
||||
if new_end_byte < start_byte {
|
||||
return None;
|
||||
}
|
||||
Some(InputEdit {
|
||||
start_byte,
|
||||
old_end_byte,
|
||||
new_end_byte,
|
||||
start_position: point_at(old, start_byte),
|
||||
old_end_position: point_at(old, old_end_byte),
|
||||
new_end_position: point_at(new, new_end_byte),
|
||||
})
|
||||
}
|
||||
|
||||
/// The tree-sitter point (row, byte column) of byte `offset` in `text`.
|
||||
///
|
||||
/// tree-sitter counts rows by `\n` only. An offset past the end is clamped
|
||||
/// to the end of the text.
|
||||
fn point_at(text: &str, offset: usize) -> Point {
|
||||
let bytes = text.as_bytes();
|
||||
let prefix = bytes.get(..offset).unwrap_or(bytes);
|
||||
let row = prefix.iter().filter(|byte| **byte == b'\n').count();
|
||||
let line_start = prefix
|
||||
.iter()
|
||||
.rposition(|byte| *byte == b'\n')
|
||||
.map_or(0, |newline| newline.saturating_add(1));
|
||||
Point {
|
||||
row,
|
||||
column: prefix.len().saturating_sub(line_start),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn point_counts_rows_and_byte_columns() {
|
||||
let text = "ab\ncä\n";
|
||||
assert_eq!(point_at(text, 0), Point { row: 0, column: 0 });
|
||||
assert_eq!(point_at(text, 3), Point { row: 1, column: 0 });
|
||||
assert_eq!(point_at(text, 6), Point { row: 1, column: 3 });
|
||||
assert_eq!(point_at(text, 7), Point { row: 2, column: 0 });
|
||||
assert_eq!(point_at(text, 99), Point { row: 2, column: 0 });
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn oversized_text_is_not_parsed() {
|
||||
let mut parser = Parser::new().unwrap();
|
||||
let text = format!("<?php\n/*{}*/", "x".repeat(MAX_PARSE_BYTES));
|
||||
let tree = parser.parse(&text);
|
||||
assert!(tree.truncated());
|
||||
assert!(tree.to_sexp().is_empty());
|
||||
let shrunk = parser
|
||||
.reparse(&tree, &[Edit::delete(8..MAX_PARSE_BYTES)])
|
||||
.unwrap();
|
||||
assert!(!shrunk.truncated());
|
||||
assert_eq!(
|
||||
shrunk.to_sexp(),
|
||||
parser.parse("<?php\n/*xxxxxxxx*/").to_sexp()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejected_edit_is_an_error() {
|
||||
let mut parser = Parser::new().unwrap();
|
||||
let tree = parser.parse("<?php echo 1;");
|
||||
let error = parser.reparse(&tree, &[Edit::delete(5..99)]).unwrap_err();
|
||||
assert!(matches!(error, SyntaxError::Edit(_)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn empty_batch_reparses_to_the_same_tree() {
|
||||
let mut parser = Parser::new().unwrap();
|
||||
let tree = parser.parse("<?php function f() {}");
|
||||
let again = parser.reparse(&tree, &[]).unwrap();
|
||||
assert_eq!(again.to_sexp(), tree.to_sexp());
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue