M2.7: Structure tool window and syntax.outline

Engine: every open PHP buffer keeps a tree-sitter tree, reparsed
incrementally on a worker thread off the engine lock (200 ms debounce,
edit batches recorded by buffer.edit, full parse when the chain breaks).
syntax.outline {id} returns {items, truncated, version}; outlineChanged
{id} at most every 200 ms per buffer. Replaced trees and outlines are
freed on the worker.

UI: left stripe with Project | Structure in MainLayout, the Structure
tool window (WAI-ARIA tree, Lucide kind icons, signatures, jumps via
editor.reveal), view.toggleStructure on Mod+2.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
danny 2026-10-17 13:31:37 +02:00
parent 73819df926
commit c68932c39c
27 changed files with 2596 additions and 28 deletions

View file

@ -14,6 +14,7 @@ layer = 4
[dependencies]
lutra-index.workspace = true
lutra-search.workspace = true
lutra-syntax.workspace = true
lutra-text.workspace = true
lutra-vfs.workspace = true
serde = { version = "1.0.229", features = ["derive"] }

View file

@ -164,7 +164,9 @@ impl Command for Edit {
.map_err(|e| EngineError::invalid_params(command, e.to_string()))?;
open.version = open.version.saturating_add(1);
let (version, dirty) = (open.version, open.dirty());
journal::mark_edited(state, p.id, std::time::Instant::now());
let now = std::time::Instant::now();
journal::mark_edited(state, p.id, now);
super::syntax::record_edits(state, p.id, p.base_version, edits, now);
state.emit(Event::BufferChanged { id: p.id, version });
if dirty != was_dirty {
state.emit(Event::DirtyChanged { id: p.id, dirty });

View file

@ -9,6 +9,7 @@ mod project;
mod recovery;
pub mod search;
mod settings;
pub mod syntax;
pub mod theme;
pub mod ui;
@ -47,5 +48,6 @@ pub fn register_builtins(registry: &mut Registry) -> Result<(), EngineError> {
registry.register(Box::new(goto::File))?;
registry.register(Box::new(goto::Class))?;
registry.register(Box::new(goto::Symbol))?;
registry.register(Box::new(syntax::OutlineCommand))?;
Ok(())
}

View file

@ -0,0 +1,796 @@
//! Syntax of open buffers: the per-buffer tree-sitter parse and
//! `syntax.outline`.
//!
//! Every open PHP buffer (`.php`, `.phtml`, `.inc`) has a syntax tree that
//! follows its text. Parsing never runs under the engine lock: a 1 MB file
//! takes about half a second to parse from scratch and 70–130 ms for its
//! outline (M2.1). Instead:
//! - `buffer.edit` records each edit batch ([`record_edits`]), a cheap copy
//! of the byte-offset edits it applied.
//! - [`pump`] (called by [`crate::Engine::pump_vfs`]) notices buffers whose
//! version moved past their tree and, once a buffer has been quiet for
//! [`DEBOUNCE`] (a new buffer at once), hands a snapshot of its text, the
//! last tree and the recorded batches to one worker thread. At most one
//! job per buffer is in flight.
//! - The worker reparses the old tree batch by batch
//! ([`lutra_syntax::Parser::reparse`]) and checks the result against the
//! snapshot; when no tree, no unbroken chain of batches (a reload from
//! disk or a restore changed the text without `buffer.edit`, or more than
//! [`MAX_PENDING_BATCHES`] batches piled up) or a mismatch remains, it
//! parses the snapshot from scratch. It then builds the outline as JSON.
//! - The worker also compares the new outline with the previous one, so
//! the engine never walks a large outline under its lock.
//! - The next pump stores the tree and outline and, if the outline changed,
//! queues [`Event::OutlineChanged`], at most once per
//! [`MIN_EVENT_INTERVAL`] per buffer. The first outline of a buffer is
//! always announced. Replaced trees and outlines (and those of closed
//! buffers) go back to the worker to be freed there: dropping the outline
//! of a 1 MB file takes tens of milliseconds.
//!
//! `syntax.outline {id}` answers from the stored outline, so it never waits
//! for a parse; its `version` says which buffer version the items describe
//! (`null` before the first parse finished).
use std::collections::BTreeMap;
use std::path::Path;
use std::sync::Arc;
use std::sync::mpsc::{self, Receiver, Sender, TryRecvError};
use std::time::{Duration, Instant};
use lutra_syntax::{MAX_PARSE_BYTES, OutlineItem, Parser, Range, Tree};
use lutra_text::{Buffer, Edit};
use serde::Deserialize;
use serde_json::{Map, Value, json};
use super::params::parse;
use crate::buffers::BufferId;
use crate::{Command, CommandSpec, EngineError, Event, Outcome, State};
/// How long a buffer must be unchanged before it is reparsed.
pub const DEBOUNCE: Duration = Duration::from_millis(200);
/// Shortest time between two `outlineChanged` events of one buffer.
pub const MIN_EVENT_INTERVAL: Duration = Duration::from_millis(200);
/// Most recorded edit batches per buffer; beyond it the next parse starts
/// from scratch.
pub const MAX_PENDING_BATCHES: usize = 64;
/// Most inserted bytes recorded per buffer; beyond it the next parse starts
/// from scratch.
pub const MAX_PENDING_BYTES: usize = 8 * 1024 * 1024;
/// The syntax state of the open buffers.
#[derive(Debug, Default)]
pub struct Syntax {
/// Per PHP buffer, by id.
docs: BTreeMap<BufferId, Doc>,
/// The parse thread, started by the first pump that needs it.
worker: Option<Worker>,
/// The grammar could not be loaded; nothing is ever parsed.
unavailable: bool,
}
impl Syntax {
/// Number of buffers whose parse is running.
pub fn in_flight(&self) -> usize {
self.docs.values().filter(|d| d.in_flight.is_some()).count()
}
}
/// Channels to the parse thread.
#[derive(Debug)]
struct Worker {
/// Jobs for the thread; dropping it ends the thread.
jobs: Sender<Message>,
/// Finished parses.
done: Receiver<Done>,
}
/// The outline of one buffer version, as `syntax.outline` returns it.
#[derive(Debug, Clone)]
struct Outline {
/// Buffer version the items describe.
version: u64,
/// The items as JSON, shared with the worker for its comparison.
items: Arc<Value>,
/// The text was too large to parse.
truncated: bool,
}
/// One edit batch as `buffer.edit` applied it.
#[derive(Debug, Clone)]
struct Batch {
/// Version of the text the batch applies to.
base: u64,
/// Byte-offset edits with [`Buffer::apply`] semantics.
edits: Vec<Edit>,
}
/// Syntax state of one buffer.
#[derive(Debug, Default)]
struct Doc {
/// The newest tree and the version it describes.
tree: Option<(u64, Tree)>,
/// The newest outline.
outline: Option<Outline>,
/// Batches recorded after the tree's version, oldest first.
batches: Vec<Batch>,
/// Inserted bytes in [`Doc::batches`].
batch_bytes: usize,
/// The buffer version the engine last saw.
seen_version: u64,
/// When the text last changed; `None` for a new buffer (parse at once).
changed_at: Option<Instant>,
/// Version of the parse running on the worker.
in_flight: Option<u64>,
/// Version of the last parse that failed, so it is not retried.
failed: Option<u64>,
/// The outline changed and `outlineChanged` is still to be sent.
announce: bool,
/// When `outlineChanged` was last sent.
last_event: Option<Instant>,
}
/// What the engine sends the worker.
enum Message {
/// Parse a buffer.
Parse(Job),
/// Free trees and outlines the engine no longer needs.
Retire(Vec<Retired>),
}
/// A replaced tree or outline, freed on the worker.
type Retired = Box<dyn std::any::Any + Send>;
/// A parse for the worker.
struct Job {
/// The buffer.
id: BufferId,
/// The buffer version of `text`.
version: u64,
/// The text to parse (a rope snapshot).
text: Buffer,
/// The previous tree to reparse from, if any.
base: Option<Tree>,
/// The batches leading from `base` to `text`.
batches: Vec<Vec<Edit>>,
/// The outline shown now, to tell whether the new one differs.
previous: Option<Outline>,
}
/// A finished parse.
struct Done {
/// The buffer.
id: BufferId,
/// The buffer version parsed.
version: u64,
/// The tree, its outline and whether the outline differs from the
/// job's previous one; `None` if the parse panicked.
result: Option<(Tree, Outline, bool)>,
}
/// Whether a project-relative path is parsed as PHP.
fn is_php(path: &Path) -> bool {
path.extension().and_then(|e| e.to_str()).is_some_and(|e| {
["php", "phtml", "inc"]
.iter()
.any(|x| e.eq_ignore_ascii_case(x))
})
}
/// Records an edit batch that `buffer.edit` applied to buffer `id`, moving
/// it from version `base` to `base + 1`. Buffers that are not PHP or have no
/// parse yet need no record. A batch over the caps clears the record, so
/// the next parse starts from scratch.
pub fn record_edits(state: &mut State, id: BufferId, base: u64, edits: Vec<Edit>, now: Instant) {
let Some(doc) = state.syntax.docs.get_mut(&id) else {
return;
};
doc.changed_at = Some(now);
doc.seen_version = base.saturating_add(1);
if doc.tree.is_none() && doc.in_flight.is_none() {
return;
}
let bytes: usize = edits.iter().map(|e| e.text.len()).sum();
doc.batch_bytes = doc.batch_bytes.saturating_add(bytes);
doc.batches.push(Batch { base, edits });
if doc.batches.len() > MAX_PENDING_BATCHES || doc.batch_bytes > MAX_PENDING_BYTES {
doc.batches.clear();
doc.batch_bytes = 0;
}
}
/// Moves finished parses into the state, starts due parses on the worker
/// and queues `outlineChanged` events. See the module docs.
pub fn pump(state: &mut State, now: Instant) {
let mut retired = Vec::new();
drain(state, &mut retired);
sync_docs(state, now, &mut retired);
schedule(state, now);
announce(state, now);
retire(state, retired);
}
/// Hands trees and outlines to the worker to free. Without a worker, or
/// when the send fails because the worker vanished (the failed send hands
/// the items back, so ignoring it frees them here), they are freed on the
/// caller's thread.
fn retire(state: &mut State, retired: Vec<Retired>) {
if retired.is_empty() {
return;
}
if let Some(worker) = state.syntax.worker.as_ref() {
let _ = worker.jobs.send(Message::Retire(retired));
}
}
/// Takes finished parses from the worker. A vanished worker is forgotten,
/// so the next schedule starts a new one; parses it held are started again.
/// Replaced trees and outlines are queued in `retired`.
fn drain(state: &mut State, retired: &mut Vec<Retired>) {
let syntax = &mut state.syntax;
let Some(worker) = syntax.worker.as_ref() else {
return;
};
loop {
match worker.done.try_recv() {
Ok(done) => {
let Some(doc) = syntax.docs.get_mut(&done.id) else {
if let Some((tree, outline, _)) = done.result {
retired.push(Box::new(tree) as Retired);
retired.push(Box::new(outline) as Retired);
}
continue;
};
if doc.in_flight == Some(done.version) {
doc.in_flight = None;
}
let Some((tree, outline, changed)) = done.result else {
doc.failed = Some(done.version);
continue;
};
if doc.tree.as_ref().is_some_and(|(v, _)| *v > done.version) {
retired.push(Box::new(tree) as Retired);
retired.push(Box::new(outline) as Retired);
continue;
}
if let Some((_, old)) = doc.tree.replace((done.version, tree)) {
retired.push(Box::new(old) as Retired);
}
let mut kept = 0;
doc.batches.retain(|b| {
let keep = b.base >= done.version;
if keep {
kept += b.edits.iter().map(|e| e.text.len()).sum::<usize>();
}
keep
});
doc.batch_bytes = kept;
doc.announce |= changed;
if let Some(old) = doc.outline.replace(outline) {
retired.push(Box::new(old) as Retired);
}
}
Err(TryRecvError::Empty) => break,
Err(TryRecvError::Disconnected) => {
syntax.worker = None;
for doc in syntax.docs.values_mut() {
doc.in_flight = None;
}
break;
}
}
}
}
/// Adds a doc for every open PHP buffer, drops docs of closed buffers
/// (their tree and outline go to `retired`) and notes version changes that
/// `buffer.edit` did not record.
fn sync_docs(state: &mut State, now: Instant, retired: &mut Vec<Retired>) {
let buffers = &state.buffers;
let docs = &mut state.syntax.docs;
let gone: Vec<BufferId> = docs
.keys()
.copied()
.filter(|id| !buffers.get(*id).is_some_and(|b| is_php(&b.path)))
.collect();
for id in gone {
if let Some(doc) = docs.remove(&id) {
retired.extend(doc.tree.map(|(_, tree)| Box::new(tree) as Retired));
retired.extend(doc.outline.map(|o| Box::new(o) as Retired));
}
}
for id in buffers.ids() {
let Some(open) = buffers.get(id) else {
continue;
};
if !is_php(&open.path) {
continue;
}
let doc = docs.entry(id).or_insert_with(|| Doc {
seen_version: open.version,
..Doc::default()
});
if doc.seen_version != open.version {
doc.seen_version = open.version;
doc.changed_at = Some(now);
}
}
}
/// Starts a parse for every buffer whose tree is behind its text and that
/// has been quiet for [`DEBOUNCE`].
fn schedule(state: &mut State, now: Instant) {
let State {
buffers, syntax, ..
} = state;
if syntax.unavailable {
return;
}
let mut jobs = Vec::new();
for (&id, doc) in syntax.docs.iter_mut() {
let Some(open) = buffers.get(id) else {
continue;
};
let version = open.version;
let current = doc.tree.as_ref().map(|(v, _)| *v) == Some(version);
if current || doc.in_flight.is_some() || doc.failed == Some(version) {
continue;
}
if doc
.changed_at
.is_some_and(|at| now.saturating_duration_since(at) < DEBOUNCE)
{
continue;
}
let (base, batches) = match doc.tree.as_ref() {
Some((tree_version, tree)) => match chain(&doc.batches, *tree_version, version) {
Some(batches) => (Some(tree.clone()), batches),
None => (None, Vec::new()),
},
None => (None, Vec::new()),
};
doc.in_flight = Some(version);
jobs.push(Job {
id,
version,
text: open.buffer.clone(),
base,
batches,
previous: doc.outline.clone(),
});
}
if jobs.is_empty() {
return;
}
if syntax.worker.is_none() {
match spawn() {
Ok(worker) => syntax.worker = Some(worker),
Err(Unavailable::Grammar) => {
syntax.unavailable = true;
for doc in syntax.docs.values_mut() {
doc.in_flight = None;
}
return;
}
Err(Unavailable::Thread) => {
for doc in syntax.docs.values_mut() {
doc.in_flight = None;
}
return;
}
}
}
let Some(worker) = syntax.worker.as_ref() else {
return;
};
for job in jobs {
let id = job.id;
if worker.jobs.send(Message::Parse(job)).is_err()
&& let Some(doc) = syntax.docs.get_mut(&id)
{
doc.in_flight = None;
}
}
}
/// The batches leading from version `from` to version `to`, if the record
/// holds an unbroken chain.
fn chain(batches: &[Batch], from: u64, to: u64) -> Option<Vec<Vec<Edit>>> {
let mut version = from;
let mut out = Vec::new();
for batch in batches.iter().filter(|b| b.base >= from) {
if batch.base != version {
return None;
}
out.push(batch.edits.clone());
version = version.saturating_add(1);
}
(version == to).then_some(out)
}
/// Queues `outlineChanged` for buffers with a new outline whose last event
/// is at least [`MIN_EVENT_INTERVAL`] old.
fn announce(state: &mut State, now: Instant) {
let mut ids = Vec::new();
for (&id, doc) in state.syntax.docs.iter_mut() {
let due = doc
.last_event
.is_none_or(|at| now.saturating_duration_since(at) >= MIN_EVENT_INTERVAL);
if doc.announce && due {
doc.announce = false;
doc.last_event = Some(now);
ids.push(id);
}
}
for id in ids {
state.emit(Event::OutlineChanged { id });
}
}
/// Why no worker could be started.
enum Unavailable {
/// The PHP grammar does not load; retrying cannot help.
Grammar,
/// The OS refused a thread; the next pump tries again.
Thread,
}
/// Starts the parse thread with its own parser. The thread ends when the
/// job sender is dropped, i.e. with the engine.
fn spawn() -> Result<Worker, Unavailable> {
let mut parser = Parser::new().map_err(|_| Unavailable::Grammar)?;
let (jobs, job_rx) = mpsc::channel::<Message>();
let (done_tx, done) = mpsc::channel::<Done>();
std::thread::Builder::new()
.name("lutra-syntax".into())
.spawn(move || {
for message in job_rx {
match message {
Message::Parse(job) => {
if done_tx.send(work(&mut parser, job)).is_err() {
break;
}
}
Message::Retire(retired) => drop(retired),
}
}
})
.map_err(|_| Unavailable::Thread)?;
Ok(Worker { jobs, done })
}
/// Runs one job on the worker, inside a panic boundary: a panic yields a
/// [`Done`] without a result instead of ending the thread. The parser is
/// replaced after a panic, since its state is unknown; if that fails the
/// old one is kept.
fn work(parser: &mut Parser, job: Job) -> Done {
let (id, version) = (job.id, job.version);
let result =
std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| parse_job(parser, job))).ok();
if result.is_none()
&& let Ok(fresh) = Parser::new()
{
*parser = fresh;
}
Done {
id,
version,
result,
}
}
/// Parses a job's text, incrementally when its batches lead from its base
/// tree to exactly that text, builds the outline and tells whether it
/// differs from the job's previous outline.
fn parse_job(parser: &mut Parser, job: Job) -> (Tree, Outline, bool) {
let snapshot = job.text.text();
let incremental = job.base.and_then(|mut tree| {
for batch in &job.batches {
tree = parser.reparse(&tree, batch).ok()?;
}
(tree.text().text() == snapshot).then_some(tree)
});
let tree = incremental.unwrap_or_else(|| parser.parse(&snapshot));
let items = lutra_syntax::outline(&tree);
let outline = Outline {
version: job.version,
items: Arc::new(Value::Array(items.iter().map(item_json).collect())),
truncated: tree.truncated(),
};
let changed = job
.previous
.is_none_or(|p| p.truncated != outline.truncated || p.items != outline.items);
(tree, outline, changed)
}
/// An outline item as JSON: `{kind, name, detail?, range, selectionRange,
/// children}`.
fn item_json(item: &OutlineItem) -> Value {
let mut map = Map::new();
map.insert("kind".into(), json!(item.kind.as_str()));
map.insert("name".into(), json!(item.name));
if let Some(detail) = &item.detail {
map.insert("detail".into(), json!(detail));
}
map.insert("range".into(), range_json(item.range));
map.insert("selectionRange".into(), range_json(item.selection_range));
map.insert(
"children".into(),
Value::Array(item.children.iter().map(item_json).collect()),
);
Value::Object(map)
}
/// A range as `{start: {line, column}, end: {line, column}}`.
fn range_json(range: Range) -> Value {
json!({
"start": { "line": range.start.line, "column": range.start.column },
"end": { "line": range.end.line, "column": range.end.column },
})
}
/// Params of `syntax.outline`.
#[derive(Deserialize)]
#[serde(deny_unknown_fields)]
struct OutlineParams {
/// Buffer id.
id: BufferId,
}
/// `syntax.outline`: the declarations of an open buffer.
pub struct OutlineCommand;
impl Command for OutlineCommand {
fn spec(&self) -> &CommandSpec {
&CommandSpec {
id: "syntax.outline",
label: "File Outline",
menu_path: None,
default_shortcut: None,
params: "{ id: number }; returns { items: [{ kind, name, detail?, range, selectionRange, children }], truncated, version } from the last finished parse (version null before the first; empty for non-PHP files); kind is namespace|class|interface|trait|enum|function|method|property|constant|enumCase; ranges are { start, end } of { line, column } (0-based, UTF-16 columns); event outlineChanged { id } when it changes (at most every 200 ms per buffer)",
}
}
fn enabled(&self, state: &State) -> bool {
state.has_project()
}
fn run(&self, state: &mut State, params: Value) -> Result<Outcome, EngineError> {
let command = self.spec().id;
let p: OutlineParams = parse(command, params)?;
let open = state
.buffers
.get(p.id)
.ok_or_else(|| EngineError::Failed(format!("{command}: no buffer {}", p.id)))?;
let outline = state
.syntax
.docs
.get(&p.id)
.and_then(|d| d.outline.as_ref());
let value = match outline {
Some(o) => json!({ "items": *o.items, "truncated": o.truncated, "version": o.version }),
None => json!({
"items": [],
"truncated": is_php(&open.path) && open.buffer.len_bytes() > MAX_PARSE_BYTES,
"version": Value::Null,
}),
};
Ok(Outcome::value(value))
}
}
#[cfg(test)]
mod tests {
use super::*;
use std::path::PathBuf;
use crate::OpenBuffer;
fn open(state: &mut State, path: &str, text: &str) -> BufferId {
state.buffers.insert(OpenBuffer {
buffer: Buffer::new(text),
path: PathBuf::from(path),
version: 0,
saved_version: 0,
bom: false,
})
}
fn edit(state: &mut State, id: BufferId, edits: Vec<Edit>, now: Instant) {
let open = state.buffers.get_mut(id).unwrap();
let base = open.version;
open.buffer.apply(&edits).unwrap();
open.version += 1;
record_edits(state, id, base, edits, now);
}
/// Pumps with a clock 50 ms apart per round until every doc's tree is
/// current and nothing waits to be announced.
fn settle(state: &mut State, mut now: Instant) -> (Vec<Event>, Instant) {
let mut events = Vec::new();
for _ in 0..500 {
pump(state, now);
events.extend(state.take_events());
let current = state.syntax.docs.iter().all(|(id, d)| {
let version = state.buffers.get(*id).unwrap().version;
d.tree.as_ref().map(|(v, _)| *v) == Some(version) && !d.announce
});
if current && state.syntax.in_flight() == 0 {
return (events, now);
}
std::thread::sleep(Duration::from_millis(5));
now += Duration::from_millis(50);
}
panic!("parse did not finish");
}
fn names(state: &State, id: BufferId) -> Vec<String> {
fn walk(items: &Value, out: &mut Vec<String>) {
for item in items.as_array().unwrap() {
out.push(item["name"].as_str().unwrap().to_string());
walk(&item["children"], out);
}
}
let mut out = Vec::new();
let outline = state.syntax.docs[&id].outline.as_ref().unwrap();
walk(&outline.items, &mut out);
out
}
#[test]
fn new_buffers_parse_at_once_and_announce() {
let mut state = State::default();
let id = open(&mut state, "a.php", "<?php class A { function f() {} }");
let other = open(&mut state, "readme.md", "# class B");
let (events, _) = settle(&mut state, Instant::now());
assert_eq!(events, vec![Event::OutlineChanged { id }]);
assert_eq!(names(&state, id), ["A", "f"]);
assert!(!state.syntax.docs.contains_key(&other));
}
#[test]
fn edits_are_debounced_and_reparsed_incrementally() {
let mut state = State::default();
let text = "<?php class A {}";
let id = open(&mut state, "a.php", text);
let (_, mut now) = settle(&mut state, Instant::now());
now += Duration::from_secs(1);
edit(
&mut state,
id,
vec![Edit::insert(text.len() - 1, " function g() {} ")],
now,
);
edit(&mut state, id, vec![Edit::replace(12..13, "B")], now);
pump(&mut state, now);
assert_eq!(state.syntax.in_flight(), 0, "debounced");
let doc = &state.syntax.docs[&id];
assert_eq!(doc.batches.len(), 2);
let (events, _) = settle(&mut state, now + DEBOUNCE);
assert_eq!(events, vec![Event::OutlineChanged { id }]);
assert_eq!(names(&state, id), ["B", "g"]);
let doc = &state.syntax.docs[&id];
assert!(doc.batches.is_empty());
assert_eq!(doc.outline.as_ref().unwrap().version, 2);
}
#[test]
fn unrecorded_changes_parse_from_scratch() {
let mut state = State::default();
let id = open(&mut state, "a.php", "<?php class A {}");
let (_, now) = settle(&mut state, Instant::now());
let open = state.buffers.get_mut(id).unwrap();
open.buffer = Buffer::new("<?php interface I {}");
open.version += 1;
let (events, _) = settle(&mut state, now + Duration::from_secs(1));
assert_eq!(events, vec![Event::OutlineChanged { id }]);
assert_eq!(names(&state, id), ["I"]);
}
#[test]
fn an_unchanged_outline_is_not_announced_again() {
let mut state = State::default();
let id = open(&mut state, "a.php", "<?php class A {} ");
let (_, now) = settle(&mut state, Instant::now());
let now = now + Duration::from_secs(1);
edit(&mut state, id, vec![Edit::insert(17, "\n")], now);
let (events, _) = settle(&mut state, now + DEBOUNCE);
assert!(events.is_empty(), "{events:?}");
assert_eq!(state.syntax.docs[&id].outline.as_ref().unwrap().version, 1);
}
#[test]
fn events_are_rate_limited_per_buffer() {
let mut state = State::default();
let id = open(&mut state, "a.php", "<?php class A {}");
let start = Instant::now();
pump(&mut state, start);
while state.syntax.in_flight() > 0 {
std::thread::sleep(Duration::from_millis(2));
drain(&mut state, &mut Vec::new());
}
announce(&mut state, start);
assert_eq!(state.take_events(), vec![Event::OutlineChanged { id }]);
let doc = state.syntax.docs.get_mut(&id).unwrap();
doc.announce = true;
announce(&mut state, start + Duration::from_millis(100));
assert!(state.take_events().is_empty());
announce(&mut state, start + MIN_EVENT_INTERVAL);
assert_eq!(state.take_events(), vec![Event::OutlineChanged { id }]);
}
#[test]
fn too_many_batches_fall_back_to_a_full_parse() {
let mut state = State::default();
let id = open(&mut state, "a.php", "<?php class A {}");
let (_, now) = settle(&mut state, Instant::now());
for _ in 0..=MAX_PENDING_BATCHES {
edit(&mut state, id, vec![Edit::insert(6, " ")], now);
}
assert!(state.syntax.docs[&id].batches.is_empty());
let (events, _) = settle(&mut state, now + DEBOUNCE);
assert_eq!(events, vec![Event::OutlineChanged { id }]);
assert_eq!(names(&state, id), ["A"]);
}
#[test]
fn closed_buffers_are_dropped() {
let mut state = State::default();
let id = open(&mut state, "a.php", "<?php function f() {}");
let (_, now) = settle(&mut state, Instant::now());
state.buffers.remove(id);
pump(&mut state, now);
assert!(state.syntax.docs.is_empty());
assert!(state.take_events().is_empty());
}
#[test]
fn chain_needs_every_batch() {
let batch = |base| Batch {
base,
edits: Vec::new(),
};
assert_eq!(chain(&[batch(2), batch(3)], 2, 4).map(|c| c.len()), Some(2));
assert_eq!(
chain(&[batch(1), batch(2), batch(3)], 2, 4).map(|c| c.len()),
Some(2)
);
assert!(chain(&[batch(3)], 2, 4).is_none());
assert!(chain(&[batch(2)], 2, 4).is_none());
assert_eq!(chain(&[], 4, 4).map(|c| c.len()), Some(0));
}
#[test]
fn item_json_has_the_contract_shape() {
let mut parser = Parser::new().unwrap();
let tree = parser.parse("<?php\nclass A {\n public function f(int $x): int {}\n}\n");
let items: Vec<Value> = lutra_syntax::outline(&tree).iter().map(item_json).collect();
let class = &items[0];
assert_eq!(class["kind"], "class");
assert!(class.get("detail").is_none());
assert_eq!(class["range"]["start"], json!({"line": 1, "column": 0}));
let method = &class["children"][0];
assert_eq!(method["kind"], "method");
assert_eq!(method["detail"], "(int $x): int");
assert_eq!(
method["selectionRange"]["start"],
json!({"line": 2, "column": 18})
);
}
#[test]
fn php_extensions() {
for (path, php) in [
("a.php", true),
("B.PHP", true),
("v.phtml", true),
("c.inc", true),
("d.js", false),
("php", false),
] {
assert_eq!(is_php(Path::new(path)), php, "{path}");
}
}
}

View file

@ -132,7 +132,8 @@ impl Engine {
/// changed directories and conflicts), writes due recovery-journal
/// entries (see [`crate::journal`]), moves finished search results into
/// events (see [`cmd::search`]), merges index updates (see
/// [`cmd::goto`]) and returns every queued event.
/// [`cmd::goto`]), stores finished parses and starts due ones (see
/// [`cmd::syntax`]) and returns every queued event.
/// Hosts call it regularly, e.g. every 100 ms from a background thread.
///
/// # Errors
@ -145,6 +146,7 @@ impl Engine {
crate::journal::tick(state, std::time::Instant::now());
cmd::search::pump(state);
cmd::goto::sync(state);
cmd::syntax::pump(state, std::time::Instant::now());
Ok(())
})?;
Ok(state.take_events())
@ -387,6 +389,7 @@ mod tests {
"settings.list",
"settings.reset",
"settings.set",
"syntax.outline",
"test.disabled",
"theme.list",
"theme.systemPreference",

View file

@ -187,6 +187,13 @@ pub enum Event {
/// Wall time of the build in milliseconds.
elapsed_ms: u64,
},
/// The outline of a buffer changed; refetch it with `syntax.outline`.
/// Sent at most every 200 ms per buffer, and once after a buffer's
/// first parse.
OutlineChanged {
/// Buffer id.
id: u64,
},
}
/// One match of [`Event::SearchResults`]. Positions are 0-based; columns
@ -361,4 +368,12 @@ mod tests {
json!({"kind": "recoveryResolved", "paths": ["src/a.php"]})
);
}
#[test]
fn outline_changed_shape() {
assert_eq!(
serde_json::to_value(Event::OutlineChanged { id: 4 }).unwrap(),
json!({"kind": "outlineChanged", "id": 4})
);
}
}

View file

@ -39,6 +39,8 @@ pub struct State {
pub searches: crate::cmd::search::Searches,
/// The symbol index of the open project (`index.status`, `goto.*`).
pub index: crate::cmd::goto::IndexState,
/// Syntax trees and outlines of open PHP buffers (`syntax.outline`).
pub syntax: crate::cmd::syntax::Syntax,
/// Number of notifications emitted, for their ids.
notifications: u64,
/// Events queued for the host, oldest first.

View file

@ -0,0 +1,7 @@
# Seeds for failure cases proptest has generated in the past. It is
# automatically read and these particular cases re-run before any
# novel cases are generated.
#
# It is recommended to check this file in to source control so that
# everyone who runs the test benefits from these saved cases.
cc 487913cee37be422b8983c4418730212af1faa10456b87b98572ae2b7de4a9b3 # shrinks to edits = [(0.0, 0.0, "")]

View file

@ -0,0 +1,239 @@
//! `syntax.outline` and `outlineChanged` through the engine: the outline
//! follows `file.open` and `buffer.edit` via the pump, random edit sequences
//! end at the outline of a fresh parse, and (ignored) timings on a 1 MB file.
#![allow(
clippy::unwrap_used,
clippy::expect_used,
clippy::panic,
clippy::indexing_slicing
)]
use std::fs;
use std::time::{Duration, Instant};
use lutra_engine::{Engine, Event};
use proptest::prelude::*;
use serde_json::{Value, json};
use tempfile::TempDir;
/// A temp folder with one file, opened in a fresh engine with the file in a
/// buffer; returns the buffer id.
fn open_file(name: &str, text: &str) -> (TempDir, Engine, u64) {
let dir = tempfile::tempdir().unwrap();
fs::write(dir.path().join(name), text).unwrap();
let mut engine = Engine::new().unwrap();
engine
.execute("project.open", json!({"path": dir.path()}))
.unwrap();
let out = engine.execute("file.open", json!({"path": name})).unwrap();
let id = out.value["id"].as_u64().unwrap();
(dir, engine, id)
}
/// Pumps until the outline describes `version`; returns the outline and
/// the number of `outlineChanged` events seen.
fn outline_at(engine: &mut Engine, id: u64, version: u64) -> (Value, usize) {
let deadline = Instant::now() + Duration::from_secs(20);
let mut changed = 0;
while Instant::now() < deadline {
changed += engine
.pump_vfs()
.unwrap()
.iter()
.filter(|e| **e == Event::OutlineChanged { id })
.count();
let out = engine.execute("syntax.outline", json!({"id": id})).unwrap();
if out.value["version"] == version {
return (out.value, changed);
}
std::thread::sleep(Duration::from_millis(10));
}
panic!("no outline for version {version} within 20 s");
}
/// Names of the items, depth first.
fn names(items: &Value) -> Vec<String> {
let mut out = Vec::new();
for item in items.as_array().unwrap() {
out.push(item["name"].as_str().unwrap().to_string());
out.extend(names(&item["children"]));
}
out
}
/// Names of a fresh parse's outline, depth first.
fn fresh_names(text: &str) -> Vec<String> {
fn walk(items: &[lutra_syntax::OutlineItem], out: &mut Vec<String>) {
for item in items {
out.push(item.name.clone());
walk(&item.children, out);
}
}
let tree = lutra_syntax::Parser::new().unwrap().parse(text);
let mut out = Vec::new();
walk(&lutra_syntax::outline(&tree), &mut out);
out
}
#[test]
fn outline_follows_open_and_edit() {
let text = "<?php\nfunction a() {}\n";
let (_dir, mut engine, id) = open_file("a.php", text);
let first = engine.execute("syntax.outline", json!({"id": id})).unwrap();
assert_eq!(
first.value,
json!({"items": [], "truncated": false, "version": null})
);
let (outline, changed) = outline_at(&mut engine, id, 1);
assert_eq!(names(&outline["items"]), ["a"]);
assert_eq!(changed, 1);
let edit = engine
.execute(
"buffer.edit",
json!({"id": id, "baseVersion": 1, "edits": [{"from": 6, "to": 6, "insert": "function b() {}\n"}]}),
)
.unwrap();
assert!(
!edit
.events
.iter()
.any(|e| matches!(e, Event::OutlineChanged { .. })),
"outlines only come from the pump"
);
let (outline, _) = outline_at(&mut engine, id, 2);
assert_eq!(names(&outline["items"]), ["b", "a"]);
engine
.execute("buffer.close", json!({"id": id, "force": true}))
.unwrap();
assert!(engine.pump_vfs().unwrap().is_empty());
assert_eq!(
engine
.execute("syntax.outline", json!({"id": id}))
.unwrap_err()
.code(),
"failed"
);
}
/// One random edit: a position and a length as fractions of the text, and
/// what to insert.
fn edit_strategy() -> impl Strategy<Value = (f64, f64, String)> {
let insert = prop_oneof![
Just(String::new()),
Just("function f() {}\n".to_string()),
Just("class C { const X = 1; }\n".to_string()),
Just("{".to_string()),
Just("}".to_string()),
Just("\u{e4}\u{20ac}\u{1f600}".to_string()),
"[a-z$(){};\\n ]{0,12}",
];
(0.0..1.0f64, 0.0..0.2f64, insert)
}
/// Whether a UTF-16 unit is a low surrogate (second half of a pair).
fn low_surrogate(unit: u16) -> bool {
(0xDC00..0xE000).contains(&unit)
}
proptest! {
#![proptest_config(ProptestConfig::with_cases(24))]
/// Random edits through `buffer.edit` never fail the pump and end at the
/// outline of a fresh parse of the final text whenever that text is free
/// of syntax errors (tree-sitter's recovery may differ on broken code).
#[test]
fn random_edits_end_at_a_fresh_outline(edits in proptest::collection::vec(edit_strategy(), 1..12)) {
let start = "<?php\nnamespace N;\nclass A {\n function m() {}\n}\n";
let (_dir, mut engine, id) = open_file("a.php", start);
outline_at(&mut engine, id, 1);
let mut text: Vec<u16> = start.encode_utf16().collect();
let mut version = 1u64;
for (at, len, insert) in edits {
let mut from = (at * text.len() as f64) as usize;
while from > 0 && from < text.len() && low_surrogate(text[from]) {
from -= 1;
}
let mut to = (from + (len * text.len() as f64) as usize).min(text.len());
while to < text.len() && low_surrogate(text[to]) {
to += 1;
}
engine.execute(
"buffer.edit",
json!({"id": id, "baseVersion": version, "edits": [{"from": from, "to": to, "insert": insert}]}),
).unwrap();
text.splice(from..to, insert.encode_utf16());
version += 1;
if version.is_multiple_of(3) {
engine.pump_vfs().unwrap();
}
}
let final_text = String::from_utf16(&text).unwrap();
let (outline, _) = outline_at(&mut engine, id, version);
let tree = lutra_syntax::Parser::new().unwrap().parse(&final_text);
if !tree.has_errors() {
prop_assert_eq!(names(&outline["items"]), fresh_names(&final_text));
}
}
}
/// Generated PHP of about `bytes` bytes: classes with methods and
/// properties.
fn big_php(bytes: usize) -> String {
let mut text = String::from("<?php\nnamespace App\\Big;\n\n");
let mut n = 0;
while text.len() < bytes {
text.push_str(&format!(
"class C{n} extends Base implements I\n{{\n private ?string $name = null;\n public const LIMIT = {n};\n\n public function find(int $id, string ...$tags): ?Product\n {{\n return $this->repo->find($id, $tags) ?? null;\n }}\n\n public function save(Product $p): void\n {{\n $this->repo->save($p);\n }}\n}}\n\n"
));
n += 1;
}
text
}
/// Timings on a 1 MB file: `buffer.edit` and `pump_vfs` under the engine
/// lock, and the time from an edit to its `outlineChanged` (the debounce,
/// the worker's reparse and outline, and the pump interval). Run with
/// `cargo test --release -p lutra-engine --test syntax -- --ignored --nocapture`.
#[test]
#[ignore = "timing; run in release on an idle machine"]
fn perf_one_megabyte() {
let text = big_php(1024 * 1024);
let (_dir, mut engine, id) = open_file("big.php", &text);
let started = Instant::now();
outline_at(&mut engine, id, 1);
println!("first outline after {:?}", started.elapsed());
let mut edit_max = Duration::ZERO;
let mut pump_max = Duration::ZERO;
let mut to_event = Vec::new();
let at = text.len() - 2;
for version in 1..=10u64 {
let t = Instant::now();
engine
.execute(
"buffer.edit",
json!({"id": id, "baseVersion": version, "edits": [{"from": at, "to": at, "insert": format!("function f{version}() {{}}\n")}]}),
)
.unwrap();
edit_max = edit_max.max(t.elapsed());
let edited = Instant::now();
loop {
let t = Instant::now();
let events = engine.pump_vfs().unwrap();
pump_max = pump_max.max(t.elapsed());
if events.contains(&Event::OutlineChanged { id }) {
to_event.push(edited.elapsed());
break;
}
std::thread::sleep(Duration::from_millis(5));
}
std::thread::sleep(Duration::from_millis(200));
}
let t = Instant::now();
let out = engine.execute("syntax.outline", json!({"id": id})).unwrap();
let outline_time = t.elapsed();
let count = names(&out.value["items"]).len();
println!("buffer.edit max {edit_max:?}, pump_vfs max {pump_max:?}");
println!("syntax.outline {outline_time:?} for {count} items");
println!("edit to outlineChanged: {to_event:?}");
}