M2.1: tree-sitter PHP in crates/syntax

Parser (parse, incremental reparse from lutra_text edits), outline with
UTF-16 ranges and short signatures, folding ranges, 8 MiB parse cap.
Snapshot tests for five PHP 8.3 fixtures, proptests (random bytes,
reparse equals fresh parse, undo round trip), ignored release timing
test. Docs: architecture Syntax section, parity evidence, G4 note.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
danny 2026-10-16 22:06:21 +02:00
parent d12a517009
commit a5236acd7e
25 changed files with 1768 additions and 6 deletions

View file

@ -0,0 +1,67 @@
<?php
declare(strict_types=1);
namespace Shop\Catalog {
use Shop\Money\Price;
interface Sellable extends \Stringable, \Countable
{
public const string CURRENCY = 'EUR';
public function price(): Price;
}
final class Product implements Sellable
{
private array $tags = [], $notes = [];
public function __construct(
public readonly int $id,
protected ?string $title = null,
private array &$cache = [],
string $unused = '',
) {
}
public function find(int $id): ?Product
{
return $id === $this->id ? $this : null;
}
public static function tagged(string ...$tags): static
{
return new static(0);
}
public function price(): Price
{
return new Price(0);
}
public function __toString(): string
{
return (string) $this->title;
}
public function count(): int
{
return \count($this->tags);
}
}
}
namespace Shop\Orders {
function total(array $lines, float $vat = 0.19): float
{
return array_sum($lines) * (1 + $vat);
}
const MAX_LINES = 100, MIN_LINES = 1;
}
namespace {
function bootstrap(): void
{
}
}

View file

@ -0,0 +1,36 @@
# outline
namespace Shop\Catalog @4:0-51:1 sel 4:10-4:22
interface Sellable extends \Stringable, \Countable @7:4-12:5 sel 7:14-7:22
constant CURRENCY string @9:8-9:45 sel 9:28-9:36
method price (): Price @11:8-11:39 sel 11:24-11:29
class Product implements Sellable @14:4-50:5 sel 14:16-14:23
property $tags array @16:22-16:32 sel 16:22-16:27
property $notes array @16:34-16:45 sel 16:34-16:40
method __construct (int $id, ?string $title, array &$cache, string $unused) @18:8-24:9 sel 18:24-18:35
property $id int @19:12-19:35 sel 19:32-19:35
property $title ?string @20:12-20:43 sel 20:30-20:36
property $cache array @21:12-21:38 sel 21:27-21:33
method find (int $id): ?Product @26:8-29:9 sel 26:24-26:28
method tagged (string ...$tags): static @31:8-34:9 sel 31:31-31:37
method price (): Price @36:8-39:9 sel 36:24-36:29
method __toString (): string @41:8-44:9 sel 41:24-41:34
method count (): int @46:8-49:9 sel 46:24-46:29
namespace Shop\Orders @53:0-60:1 sel 53:10-53:21
function total (array $lines, float $vat): float @54:4-57:5 sel 54:13-54:18
constant MAX_LINES @59:10-59:25 sel 59:10-59:19
constant MIN_LINES @59:27-59:40 sel 59:27-59:36
function bootstrap (): void @63:4-65:5 sel 63:13-63:22
# folding
4:23-51:1
8:4-12:5
15:4-50:5
23:10-24:9
27:8-29:9
32:8-34:9
37:8-39:9
42:8-44:9
47:8-49:9
53:22-60:1
55:4-57:5
62:10-66:1
64:4-65:5

28
crates/syntax/tests/fixtures/broken.php vendored Normal file
View file

@ -0,0 +1,28 @@
<?php
namespace Broken;
class Complete
{
public function ok(): bool
{
return true;
}
}
class Half extends
{
public function first(int $a
public function second(): void
{
if ($x {
}
}
function after_errors(): int
{
return 1;
}
class

View file

@ -0,0 +1,12 @@
# outline
namespace Broken @2:0-27:5 sel 2:10-2:16
class Complete @4:0-10:1 sel 4:6-4:14
method ok (): bool @6:4-9:5 sel 6:20-6:22
method second (): void @14:4-20:1 sel 16:20-16:26
method after_errors (): int @22:0-25:1 sel 22:9-22:21
# folding
5:0-10:1
7:4-9:5
17:4-20:1
18:15-19:5
23:0-25:1

View file

@ -0,0 +1,39 @@
<html>
<body>
<?php
if (!function_exists('greet')) {
function greet(string $name = 'world'): string
{
return "Hello, {$name}!";
}
}
$handler = function (int $code) {
function not_listed_inside_closure() {}
return $code;
};
$double = fn(int $x): int => $x * 2;
return new class ('db') extends Migration implements Countable {
public function __construct(private string $connection)
{
}
public function up(): void
{
$inner = new class {
public function hidden(): void {}
};
}
public function count(): int
{
return 0;
}
};
?>
<p><?= greet() ?></p>
</body>
</html>

View file

@ -0,0 +1,16 @@
# outline
function greet (string $name): string @5:4-8:5 sel 5:13-5:18
class class@anonymous extends Migration implements Countable @18:11-34:1 sel 18:11-18:16
method __construct (string $connection) @19:4-21:5 sel 19:20-19:31
property $connection string @19:32-19:58 sel 19:47-19:58
method up (): void @23:4-28:5 sel 23:20-23:22
method count (): int @30:4-33:5 sel 30:20-30:25
# folding
4:31-9:1
6:4-8:5
11:32-14:1
18:63-34:1
20:4-21:5
24:4-28:5
25:27-27:9
31:4-33:5

View file

@ -0,0 +1,64 @@
<?php
namespace App\Models;
use App\Contracts\HasLabel;
use App\Support\Str;
use function App\Support\slugify;
#[\Attribute(\Attribute::TARGET_CLASS)]
final class Entity
{
}
/**
* Base model for every entity.
*/
#[Entity]
abstract class Model
{
protected static int $instances = 0;
#[\Deprecated]
abstract public function key(): string|int;
public function &attributes(): array
{
static $cache = [];
return $cache;
}
}
trait Timestamps
{
public ?\DateTimeImmutable $createdAt = null;
public function touch(\DateTimeImmutable $at = new \DateTimeImmutable()): void
{
$this->createdAt = $at;
}
}
namespace App\Enums;
enum Status: string implements \App\Contracts\HasLabel
{
case Draft = 'draft';
case Published = 'published';
const DEFAULT = self::Draft;
public function label(): string
{
return match ($this) {
self::Draft => 'Draft',
self::Published => 'Published',
};
}
}
enum Suit
{
case Hearts;
case Spades;
}

View file

@ -0,0 +1,31 @@
# outline
namespace App\Models @2:0-39:1 sel 2:10-2:20
class Entity @8:0-11:1 sel 9:12-9:18
class Model @16:0-29:1 sel 17:15-17:20
property $instances int @19:4-19:40 sel 19:25-19:35
method key (): string|int @21:4-22:47 sel 22:29-22:32
method attributes (): array @24:4-28:5 sel 24:21-24:31
trait Timestamps @31:0-39:1 sel 31:6-31:16
property $createdAt ?\DateTimeImmutable @33:4-33:49 sel 33:31-33:41
method touch (\DateTimeImmutable $at): void @35:4-38:5 sel 35:20-35:25
namespace App\Enums @41:0-63:1 sel 41:10-41:19
enum Status : string implements \App\Contracts\HasLabel @43:0-57:1 sel 43:5-43:11
enumCase Draft 'draft' @45:4-45:25 sel 45:9-45:14
enumCase Published 'published' @46:4-46:33 sel 46:9-46:18
constant DEFAULT @48:4-48:32 sel 48:10-48:17
method label (): string @50:4-56:5 sel 50:20-50:25
enum Suit @59:0-63:1 sel 59:5-59:9
enumCase Hearts @61:4-61:16 sel 61:9-61:15
enumCase Spades @62:4-62:16 sel 62:9-62:15
# folding
4:0-6:33
10:0-11:1
13:0-15:3
18:0-29:1
25:4-28:5
32:0-39:1
36:4-38:5
44:0-57:1
51:4-56:5
52:29-55:9
60:0-63:1

View file

@ -0,0 +1,4 @@
<?php
/** Prüfung — 🦦 otters use tools. */
const ÄRGER = '🦦🦦'; class Café { public function naïve(string $straße): string { return $straße; } }
$otters = "🦦🦦🦦"; function grüße(string $gruß = "🦦"): void {}

View file

@ -0,0 +1,6 @@
# outline
constant ÄRGER @2:0-2:21 sel 2:6-2:11
class Café @2:22-2:102 sel 2:28-2:32
method naïve (string $straße): string @2:35-2:100 sel 2:51-2:56
function grüße (string $gruß): void @3:20-3:64 sel 3:29-3:34
# folding

112
crates/syntax/tests/perf.rs Normal file
View file

@ -0,0 +1,112 @@
//! Parse timing for a generated 1 MB PHP file (budget: under 50 ms).
//!
//! Ignored by default because timings in debug builds mean nothing. Run in
//! release on an idle machine:
//! `cargo test --release -p lutra-syntax --test perf -- --ignored --nocapture`.
#![allow(
clippy::unwrap_used,
clippy::indexing_slicing,
clippy::string_slice,
reason = "tests may panic"
)]
use std::time::{Duration, Instant};
use lutra_syntax::{Parser, folding, outline};
use lutra_text::Edit;
/// Generates a PHP file of at least `bytes` bytes made of namespaced
/// classes with properties, a promoted constructor, methods, arrays and
/// doc comments.
fn generate(bytes: usize) -> String {
let mut text = String::from(
"<?php\n\ndeclare(strict_types=1);\n\nnamespace Bench\\Generated;\n\nuse Bench\\Support\\Collection;\n\n",
);
let mut index = 0usize;
while text.len() < bytes {
text.push_str(&format!(
r#"/**
* Generated class number {index}.
*/
final class Service{index} implements \Countable
{{
public const int LIMIT = {index};
private array $items = ['a' => 1, 'b' => [2, 3], 'c' => "four {index}"];
public function __construct(private readonly Collection $collection, protected ?string $name = null)
{{
}}
public function find(int $id, string ...$tags): ?array
{{
foreach ($this->items as $key => $value) {{
if ($key === $id && \in_array($value, $tags, true)) {{
return ['id' => $id, 'value' => $value];
}}
}}
return match ($id) {{
0 => null,
default => $this->collection->map(fn (int $x): int => $x * {index})->all(),
}};
}}
public function count(): int
{{
return \count($this->items) + self::LIMIT;
}}
}}
"#
));
index += 1;
}
text
}
/// The fastest of `runs` timings of `work`.
fn best_of(runs: usize, mut work: impl FnMut()) -> Duration {
(0..runs)
.map(|_| {
let started = Instant::now();
work();
started.elapsed()
})
.min()
.unwrap()
}
#[test]
#[ignore = "timing; run in release"]
fn parse_one_megabyte() {
let text = generate(1024 * 1024);
let mut parser = Parser::new().unwrap();
let tree = parser.parse(&text);
assert!(!tree.has_errors());
let parse = best_of(5, || {
parser.parse(&text);
});
let outline_time = best_of(5, || {
outline(&tree);
});
let folding_time = best_of(5, || {
folding(&tree);
});
let middle = text.len() / 2;
let middle = (middle..text.len())
.find(|offset| text.is_char_boundary(*offset))
.unwrap();
let reparse = best_of(5, || {
parser.reparse(&tree, &[Edit::insert(middle, " ")]).unwrap();
});
println!(
"1 MB PHP ({} bytes, {} top-level items): parse {:?}, reparse after one keystroke {:?}, outline {:?}, folding {:?}",
text.len(),
outline(&tree).len(),
parse,
reparse,
outline_time,
folding_time
);
}

View file

@ -0,0 +1,8 @@
# Seeds for failure cases proptest has generated in the past. It is
# automatically read and these particular cases re-run before any
# novel cases are generated.
#
# It is recommended to check this file in to source control so that
# everyone who runs the test benefits from these saved cases.
cc c8b0a3ee622e3fac3ec69a94aebc22f23c9b3870156159ba6d9c2c5387d5eced # shrinks to fixture = Index(0), steps = [[(5318099958520493566, 19, 12), (10643632058251538265, 19, 18)], [(11062977690082286900, 37, 1)], [(3728561719478013405, 20, 10)]]
cc 3129b8640200f6a3b49b6579ba02346c578c68a5fb37bf929f93fb1aa0f4ef9b # shrinks to fixture = Index(14757395258967641293), steps = [[(14373769676898819633, 13, 21)], [(0, 0, 0)]]

View file

@ -0,0 +1,210 @@
//! Property tests: parsing untrusted text never panics, and an incremental
//! reparse gives the same tree, outline and folding as a fresh parse.
//!
//! tree-sitter recovers from syntax errors heuristically, and the recovery
//! may differ between an incremental and a fresh parse of the same broken
//! text. Tree equality is therefore asserted whenever the edited text
//! parses without errors, and an undo round trip must always end at the
//! fresh tree of the (error-free) original.
#![allow(
clippy::unwrap_used,
clippy::indexing_slicing,
clippy::string_slice,
reason = "tests may panic"
)]
use std::path::Path;
use lutra_syntax::{Parser, SyntaxError, folding, outline};
use lutra_text::Edit;
use proptest::prelude::*;
/// PHP fragments edits insert, chosen to open and close constructs.
const FRAGMENTS: &[&str] = &[
"",
"{",
"}",
"(",
")",
";",
"\n",
" ",
"class X {",
"function f() {",
"$a = [",
"]",
"/*",
"*/",
"//",
"'",
"\"",
"<?php ",
"?>",
"namespace N;",
"public int $p;",
"enum E { case A; }",
"#[A]",
"=> ",
"ü",
"🦦",
"\r\n",
"<<<EOT\n",
"\nEOT;\n",
];
/// The fixture texts edits start from.
fn fixtures() -> Vec<String> {
let dir = Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures");
let mut texts: Vec<String> = std::fs::read_dir(dir)
.unwrap()
.map(|entry| entry.unwrap().path())
.filter(|path| path.extension().is_some_and(|ext| ext == "php"))
.map(|path| std::fs::read_to_string(path).unwrap())
.collect();
texts.sort();
texts
}
/// Moves `offset` down to the nearest char boundary of `text`.
fn floor_boundary(text: &str, offset: usize) -> usize {
let mut offset = offset.min(text.len());
while !text.is_char_boundary(offset) {
offset -= 1;
}
offset
}
/// A strategy for one edit as (start seed, length seed, fragment index).
fn edit_seed() -> impl Strategy<Value = (usize, usize, usize)> {
(any::<usize>(), 0usize..40, 0..FRAGMENTS.len())
}
/// Turns seeds into a valid edit on `text`.
fn edit_for(text: &str, (start, len, fragment): (usize, usize, usize)) -> Edit {
let start = floor_boundary(text, start % (text.len() + 1));
let end = floor_boundary(text, start + len);
Edit::replace(start..end.max(start), FRAGMENTS[fragment])
}
proptest! {
#![proptest_config(ProptestConfig::with_cases(256))]
#[test]
fn random_bytes_never_panic(bytes in proptest::collection::vec(any::<u8>(), 0..2048)) {
let text = String::from_utf8_lossy(&bytes);
let mut parser = Parser::new().unwrap();
let tree = parser.parse(&text);
let _ = outline(&tree);
let _ = folding(&tree);
}
#[test]
fn random_php_tokens_never_panic(parts in proptest::collection::vec(0..FRAGMENTS.len(), 0..200)) {
let text: String = parts.into_iter().map(|index| FRAGMENTS[index]).collect();
let mut parser = Parser::new().unwrap();
let tree = parser.parse(&text);
let _ = outline(&tree);
let _ = folding(&tree);
}
#[test]
fn reparse_equals_fresh_parse(
fixture in any::<prop::sample::Index>(),
steps in proptest::collection::vec(proptest::collection::vec(edit_seed(), 1..4), 1..6),
) {
let texts = fixtures();
let mut text = fixture.get(&texts).clone();
let mut parser = Parser::new().unwrap();
let mut tree = parser.parse(&text);
for step in steps {
let mut edits: Vec<Edit> = Vec::new();
for seed in step {
let edit = edit_for(&text, seed);
if edits.iter().all(|other| other.range.end <= edit.range.start || edit.range.end <= other.range.start) {
edits.push(edit);
}
}
tree = parser.reparse(&tree, &edits).unwrap();
let mut buffer = lutra_text::Buffer::new(&text);
buffer.apply(&edits).unwrap();
text = buffer.text();
let fresh = parser.parse(&text);
prop_assert_eq!(tree.text().text(), text.clone());
let _ = outline(&tree);
let _ = folding(&tree);
if !fresh.has_errors() {
prop_assert_eq!(tree.to_sexp(), fresh.to_sexp());
prop_assert_eq!(outline(&tree), outline(&fresh));
prop_assert_eq!(folding(&tree), folding(&fresh));
}
}
}
#[test]
fn undo_round_trip_restores_the_tree(
fixture in any::<prop::sample::Index>(),
seeds in proptest::collection::vec(edit_seed(), 1..12),
) {
let texts = fixtures();
let original = fixture.get(&texts).clone();
let mut parser = Parser::new().unwrap();
let expected = parser.parse(&original);
let mut tree = expected.clone();
let mut text = original.clone();
let mut undo = Vec::new();
for seed in seeds {
let edit = edit_for(&text, seed);
let removed = text[edit.range.clone()].to_owned();
let start = edit.range.start;
undo.push(Edit::replace(start..start + edit.text.len(), removed));
tree = parser.reparse(&tree, std::slice::from_ref(&edit)).unwrap();
text.replace_range(edit.range.clone(), &edit.text);
}
for edit in undo.into_iter().rev() {
tree = parser.reparse(&tree, std::slice::from_ref(&edit)).unwrap();
}
prop_assert_eq!(tree.text().text(), original);
if !expected.has_errors() {
prop_assert_eq!(tree.to_sexp(), expected.to_sexp());
prop_assert_eq!(outline(&tree), outline(&expected));
prop_assert_eq!(folding(&tree), folding(&expected));
}
}
#[test]
fn invalid_edits_are_errors(start in 0usize..200, end in 0usize..200) {
let text = "<?php class Café { function ü() {} }";
let mut parser = Parser::new().unwrap();
let tree = parser.parse(text);
let valid = start <= end
&& end <= text.len()
&& text.is_char_boundary(start)
&& text.is_char_boundary(end);
match parser.reparse(&tree, &[Edit::delete(start..end)]) {
Ok(_) => prop_assert!(valid),
Err(SyntaxError::Edit(_)) => prop_assert!(!valid),
Err(other) => prop_assert!(false, "unexpected error {other}"),
}
}
}
/// Deeply nested brackets must not exhaust the stack in the outline or
/// folding walk.
#[test]
fn deep_nesting_does_not_overflow() {
let depth = 100_000;
let text = format!("<?php $a = {}{};", "[".repeat(depth), "]".repeat(depth));
let mut parser = Parser::new().unwrap();
let tree = parser.parse(&text);
let _ = outline(&tree);
assert!(folding(&tree).is_empty());
let nested = format!(
"<?php {} class Deep {{}} {}",
"if (1) {".repeat(5_000),
"}".repeat(5_000)
);
let tree = parser.parse(&nested);
let _ = outline(&tree);
let _ = folding(&tree);
}

View file

@ -0,0 +1,96 @@
//! Outline and folding snapshots for the PHP fixtures in `tests/fixtures`.
//!
//! Each `NAME.php` has a `NAME.snap` next to it holding the rendered
//! outline and folding ranges. Run with `LUTRA_BLESS=1` to rewrite the
//! snapshots after an intended change, then review the diff.
#![allow(
clippy::unwrap_used,
clippy::indexing_slicing,
clippy::string_slice,
reason = "tests may panic"
)]
use std::fmt::Write as _;
use std::path::Path;
use lutra_syntax::{OutlineItem, Parser, Range, folding, outline};
/// Renders a range as `line:column-line:column`.
fn range(range: &Range) -> String {
format!(
"{}:{}-{}:{}",
range.start.line, range.start.column, range.end.line, range.end.column
)
}
/// Renders outline items one per line, children indented.
fn render_items(items: &[OutlineItem], depth: usize, out: &mut String) {
for item in items {
let detail = item
.detail
.as_deref()
.map(|detail| format!(" {detail}"))
.unwrap_or_default();
writeln!(
out,
"{}{} {}{} @{} sel {}",
" ".repeat(depth),
item.kind.as_str(),
item.name,
detail,
range(&item.range),
range(&item.selection_range)
)
.unwrap();
render_items(&item.children, depth + 1, out);
}
}
/// Parses `name.php` and compares the rendering with `name.snap`.
fn check(name: &str) {
let dir = Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures");
let source = std::fs::read_to_string(dir.join(format!("{name}.php"))).unwrap();
let mut parser = Parser::new().unwrap();
let tree = parser.parse(&source);
let mut rendered = String::from("# outline\n");
render_items(&outline(&tree), 0, &mut rendered);
rendered.push_str("# folding\n");
for fold in folding(&tree) {
writeln!(rendered, "{}", range(&fold)).unwrap();
}
let snapshot = dir.join(format!("{name}.snap"));
if std::env::var_os("LUTRA_BLESS").is_some() {
std::fs::write(&snapshot, &rendered).unwrap();
return;
}
let expected = std::fs::read_to_string(&snapshot)
.unwrap_or_default()
.replace("\r\n", "\n");
assert_eq!(rendered, expected, "snapshot {name}.snap differs");
}
#[test]
fn braced_namespaces() {
check("braced_namespaces");
}
#[test]
fn unbraced_namespace() {
check("unbraced_namespace");
}
#[test]
fn scripts_and_anonymous() {
check("scripts_and_anonymous");
}
#[test]
fn broken() {
check("broken");
}
#[test]
fn unicode_positions() {
check("unicode_positions");
}