M2.1: tree-sitter PHP in crates/syntax
Parser (parse, incremental reparse from lutra_text edits), outline with UTF-16 ranges and short signatures, folding ranges, 8 MiB parse cap. Snapshot tests for five PHP 8.3 fixtures, proptests (random bytes, reparse equals fresh parse, undo round trip), ignored release timing test. Docs: architecture Syntax section, parity evidence, G4 note. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
d12a517009
commit
a5236acd7e
25 changed files with 1768 additions and 6 deletions
210
crates/syntax/tests/properties.rs
Normal file
210
crates/syntax/tests/properties.rs
Normal file
|
|
@ -0,0 +1,210 @@
|
|||
//! Property tests: parsing untrusted text never panics, and an incremental
|
||||
//! reparse gives the same tree, outline and folding as a fresh parse.
|
||||
//!
|
||||
//! tree-sitter recovers from syntax errors heuristically, and the recovery
|
||||
//! may differ between an incremental and a fresh parse of the same broken
|
||||
//! text. Tree equality is therefore asserted whenever the edited text
|
||||
//! parses without errors, and an undo round trip must always end at the
|
||||
//! fresh tree of the (error-free) original.
|
||||
|
||||
#![allow(
|
||||
clippy::unwrap_used,
|
||||
clippy::indexing_slicing,
|
||||
clippy::string_slice,
|
||||
reason = "tests may panic"
|
||||
)]
|
||||
|
||||
use std::path::Path;
|
||||
|
||||
use lutra_syntax::{Parser, SyntaxError, folding, outline};
|
||||
use lutra_text::Edit;
|
||||
use proptest::prelude::*;
|
||||
|
||||
/// PHP fragments edits insert, chosen to open and close constructs.
|
||||
const FRAGMENTS: &[&str] = &[
|
||||
"",
|
||||
"{",
|
||||
"}",
|
||||
"(",
|
||||
")",
|
||||
";",
|
||||
"\n",
|
||||
" ",
|
||||
"class X {",
|
||||
"function f() {",
|
||||
"$a = [",
|
||||
"]",
|
||||
"/*",
|
||||
"*/",
|
||||
"//",
|
||||
"'",
|
||||
"\"",
|
||||
"<?php ",
|
||||
"?>",
|
||||
"namespace N;",
|
||||
"public int $p;",
|
||||
"enum E { case A; }",
|
||||
"#[A]",
|
||||
"=> ",
|
||||
"ü",
|
||||
"🦦",
|
||||
"\r\n",
|
||||
"<<<EOT\n",
|
||||
"\nEOT;\n",
|
||||
];
|
||||
|
||||
/// The fixture texts edits start from.
|
||||
fn fixtures() -> Vec<String> {
|
||||
let dir = Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures");
|
||||
let mut texts: Vec<String> = std::fs::read_dir(dir)
|
||||
.unwrap()
|
||||
.map(|entry| entry.unwrap().path())
|
||||
.filter(|path| path.extension().is_some_and(|ext| ext == "php"))
|
||||
.map(|path| std::fs::read_to_string(path).unwrap())
|
||||
.collect();
|
||||
texts.sort();
|
||||
texts
|
||||
}
|
||||
|
||||
/// Moves `offset` down to the nearest char boundary of `text`.
|
||||
fn floor_boundary(text: &str, offset: usize) -> usize {
|
||||
let mut offset = offset.min(text.len());
|
||||
while !text.is_char_boundary(offset) {
|
||||
offset -= 1;
|
||||
}
|
||||
offset
|
||||
}
|
||||
|
||||
/// A strategy for one edit as (start seed, length seed, fragment index).
|
||||
fn edit_seed() -> impl Strategy<Value = (usize, usize, usize)> {
|
||||
(any::<usize>(), 0usize..40, 0..FRAGMENTS.len())
|
||||
}
|
||||
|
||||
/// Turns seeds into a valid edit on `text`.
|
||||
fn edit_for(text: &str, (start, len, fragment): (usize, usize, usize)) -> Edit {
|
||||
let start = floor_boundary(text, start % (text.len() + 1));
|
||||
let end = floor_boundary(text, start + len);
|
||||
Edit::replace(start..end.max(start), FRAGMENTS[fragment])
|
||||
}
|
||||
|
||||
proptest! {
|
||||
#![proptest_config(ProptestConfig::with_cases(256))]
|
||||
|
||||
#[test]
|
||||
fn random_bytes_never_panic(bytes in proptest::collection::vec(any::<u8>(), 0..2048)) {
|
||||
let text = String::from_utf8_lossy(&bytes);
|
||||
let mut parser = Parser::new().unwrap();
|
||||
let tree = parser.parse(&text);
|
||||
let _ = outline(&tree);
|
||||
let _ = folding(&tree);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn random_php_tokens_never_panic(parts in proptest::collection::vec(0..FRAGMENTS.len(), 0..200)) {
|
||||
let text: String = parts.into_iter().map(|index| FRAGMENTS[index]).collect();
|
||||
let mut parser = Parser::new().unwrap();
|
||||
let tree = parser.parse(&text);
|
||||
let _ = outline(&tree);
|
||||
let _ = folding(&tree);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reparse_equals_fresh_parse(
|
||||
fixture in any::<prop::sample::Index>(),
|
||||
steps in proptest::collection::vec(proptest::collection::vec(edit_seed(), 1..4), 1..6),
|
||||
) {
|
||||
let texts = fixtures();
|
||||
let mut text = fixture.get(&texts).clone();
|
||||
let mut parser = Parser::new().unwrap();
|
||||
let mut tree = parser.parse(&text);
|
||||
for step in steps {
|
||||
let mut edits: Vec<Edit> = Vec::new();
|
||||
for seed in step {
|
||||
let edit = edit_for(&text, seed);
|
||||
if edits.iter().all(|other| other.range.end <= edit.range.start || edit.range.end <= other.range.start) {
|
||||
edits.push(edit);
|
||||
}
|
||||
}
|
||||
tree = parser.reparse(&tree, &edits).unwrap();
|
||||
let mut buffer = lutra_text::Buffer::new(&text);
|
||||
buffer.apply(&edits).unwrap();
|
||||
text = buffer.text();
|
||||
let fresh = parser.parse(&text);
|
||||
prop_assert_eq!(tree.text().text(), text.clone());
|
||||
let _ = outline(&tree);
|
||||
let _ = folding(&tree);
|
||||
if !fresh.has_errors() {
|
||||
prop_assert_eq!(tree.to_sexp(), fresh.to_sexp());
|
||||
prop_assert_eq!(outline(&tree), outline(&fresh));
|
||||
prop_assert_eq!(folding(&tree), folding(&fresh));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn undo_round_trip_restores_the_tree(
|
||||
fixture in any::<prop::sample::Index>(),
|
||||
seeds in proptest::collection::vec(edit_seed(), 1..12),
|
||||
) {
|
||||
let texts = fixtures();
|
||||
let original = fixture.get(&texts).clone();
|
||||
let mut parser = Parser::new().unwrap();
|
||||
let expected = parser.parse(&original);
|
||||
let mut tree = expected.clone();
|
||||
let mut text = original.clone();
|
||||
let mut undo = Vec::new();
|
||||
for seed in seeds {
|
||||
let edit = edit_for(&text, seed);
|
||||
let removed = text[edit.range.clone()].to_owned();
|
||||
let start = edit.range.start;
|
||||
undo.push(Edit::replace(start..start + edit.text.len(), removed));
|
||||
tree = parser.reparse(&tree, std::slice::from_ref(&edit)).unwrap();
|
||||
text.replace_range(edit.range.clone(), &edit.text);
|
||||
}
|
||||
for edit in undo.into_iter().rev() {
|
||||
tree = parser.reparse(&tree, std::slice::from_ref(&edit)).unwrap();
|
||||
}
|
||||
prop_assert_eq!(tree.text().text(), original);
|
||||
if !expected.has_errors() {
|
||||
prop_assert_eq!(tree.to_sexp(), expected.to_sexp());
|
||||
prop_assert_eq!(outline(&tree), outline(&expected));
|
||||
prop_assert_eq!(folding(&tree), folding(&expected));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn invalid_edits_are_errors(start in 0usize..200, end in 0usize..200) {
|
||||
let text = "<?php class Café { function ü() {} }";
|
||||
let mut parser = Parser::new().unwrap();
|
||||
let tree = parser.parse(text);
|
||||
let valid = start <= end
|
||||
&& end <= text.len()
|
||||
&& text.is_char_boundary(start)
|
||||
&& text.is_char_boundary(end);
|
||||
match parser.reparse(&tree, &[Edit::delete(start..end)]) {
|
||||
Ok(_) => prop_assert!(valid),
|
||||
Err(SyntaxError::Edit(_)) => prop_assert!(!valid),
|
||||
Err(other) => prop_assert!(false, "unexpected error {other}"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Deeply nested brackets must not exhaust the stack in the outline or
|
||||
/// folding walk.
|
||||
#[test]
|
||||
fn deep_nesting_does_not_overflow() {
|
||||
let depth = 100_000;
|
||||
let text = format!("<?php $a = {}{};", "[".repeat(depth), "]".repeat(depth));
|
||||
let mut parser = Parser::new().unwrap();
|
||||
let tree = parser.parse(&text);
|
||||
let _ = outline(&tree);
|
||||
assert!(folding(&tree).is_empty());
|
||||
let nested = format!(
|
||||
"<?php {} class Deep {{}} {}",
|
||||
"if (1) {".repeat(5_000),
|
||||
"}".repeat(5_000)
|
||||
);
|
||||
let tree = parser.parse(&nested);
|
||||
let _ = outline(&tree);
|
||||
let _ = folding(&tree);
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue