M2.5: symbol index and go-to commands

crates/index: walks the project with the search/VFS rules (vendor as
library), parses .php files from tree-sitter outlines on a worker pool
off the engine lock, updates incrementally from disk and buffers through
an updater thread, caps at 200,000 files, and answers fuzzy file, class
and symbol queries (nucleo-matcher, MPL-2.0 as a separate crate).

Engine: index.status, goto.file, goto.class, goto.symbol; events
indexProgress and indexReady drained by the pump; the index starts on
project.open and is cancelled on close. Tests: unit, incremental,
property (random queries and sources), engine and TCP control tests,
release timing on a generated 20k-file project (3.9-4.2 s).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
danny 2026-10-17 12:53:33 +02:00
parent f13a49064d
commit 053441d066
26 changed files with 3455 additions and 9 deletions

View file

@ -0,0 +1,170 @@
//! The symbol index over a real TCP control connection: `indexProgress`
//! and `indexReady` arrive as event lines from the pump, `index.status` and
//! the go-to commands answer.
#![allow(
clippy::unwrap_used,
clippy::expect_used,
clippy::indexing_slicing,
clippy::panic
)]
use std::io::{BufRead, BufReader, Write};
use std::net::TcpStream;
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, Ordering};
use std::time::Duration;
use lutra_testkit::EngineBackend;
use serde_json::{Value, json};
/// A client that keeps responses and event lines apart.
struct Client {
reader: BufReader<TcpStream>,
writer: TcpStream,
events: Vec<Value>,
}
impl Client {
fn connect(addr: std::net::SocketAddr) -> Self {
let stream = TcpStream::connect(addr).unwrap();
stream
.set_read_timeout(Some(Duration::from_secs(30)))
.unwrap();
Self {
writer: stream.try_clone().unwrap(),
reader: BufReader::new(stream),
events: Vec::new(),
}
}
fn line(&mut self) -> Value {
let mut line = String::new();
self.reader.read_line(&mut line).expect("line within 30 s");
serde_json::from_str(&line).unwrap()
}
fn call(&mut self, request: Value) -> Value {
self.writer
.write_all(format!("{request}\n").as_bytes())
.unwrap();
loop {
let line = self.line();
if line.get("event").is_some() {
self.events.push(line["event"].clone());
} else {
return line;
}
}
}
/// Reads event lines until `indexReady`; returns the `indexProgress`
/// events and the ready event.
fn until_ready(&mut self) -> (Vec<Value>, Value) {
let mut progress = Vec::new();
let mut pending = std::mem::take(&mut self.events).into_iter();
loop {
let event = match pending.next() {
Some(e) => e,
None => {
let line = self.line();
let Some(event) = line.get("event") else {
continue;
};
event.clone()
}
};
match event["kind"].as_str() {
Some("indexProgress") => progress.push(event),
Some("indexReady") => return (progress, event),
_ => {}
}
}
}
}
fn execute(command: &str, params: Value) -> Value {
json!({"id": command, "method": "engine.execute", "params": {"command": command, "params": params}})
}
/// Pumps the backend every 20 ms, like the desktop's pump thread, until
/// dropped.
struct Pump(Arc<AtomicBool>);
impl Pump {
fn start(backend: Arc<EngineBackend>) -> Self {
let stop = Arc::new(AtomicBool::new(false));
let flag = stop.clone();
std::thread::spawn(move || {
while !flag.load(Ordering::Relaxed) {
backend.pump();
std::thread::sleep(Duration::from_millis(20));
}
});
Self(stop)
}
}
impl Drop for Pump {
fn drop(&mut self) {
self.0.store(true, Ordering::Relaxed);
}
}
#[test]
fn index_builds_and_go_to_answers_over_tcp() {
let dir = tempfile::tempdir().unwrap();
let project = lutra_testkit::generate_php_project(dir.path(), 2_000);
let backend = Arc::new(EngineBackend::new());
let server = lutra_control::start(0, backend.clone()).unwrap();
let mut client = Client::connect(server.local_addr());
client.call(json!({"id": 1, "method": "events.subscribe"}));
let r = client.call(execute("project.open", json!({"path": dir.path()})));
assert!(r["result"]["root"].is_string(), "{r}");
let _pump = Pump::start(backend.clone());
let (progress, ready) = client.until_ready();
for event in &progress {
assert_eq!(event["total"], project.indexed_files, "{event}");
assert!(event["done"].as_u64().unwrap() <= project.indexed_files as u64);
}
assert_eq!(ready["files"], project.indexed_files);
assert!(ready["symbols"].as_u64().unwrap() > project.classes as u64);
assert!(ready["elapsedMs"].is_u64());
let r = client.call(execute("index.status", Value::Null));
assert_eq!(r["result"]["state"], "idle", "{r}");
assert_eq!(r["result"]["files"], project.indexed_files);
let r = client.call(execute(
"goto.class",
json!({"query": "Service10", "limit": 5}),
));
let items = r["result"]["items"].as_array().unwrap();
assert_eq!(r["result"]["indexing"], false);
assert_eq!(items[0]["fqn"], "App\\Module000\\Service10", "{r}");
assert_eq!(items[0]["path"], "src/Module000/Service10.php");
assert_eq!(items[0]["kind"], "class");
assert_eq!(items[0]["library"], false);
let r = client.call(execute("goto.class", json!({"query": "Widget1"})));
let first = &r["result"]["items"][0];
assert_eq!(first["library"], true, "{r}");
assert!(first["path"].as_str().unwrap().starts_with("vendor/"));
let r = client.call(execute(
"goto.symbol",
json!({"query": "Service10::handle3"}),
));
let items = r["result"]["items"].as_array().unwrap();
assert_eq!(items[0]["fqn"], "App\\Module000\\Service10::handle3", "{r}");
assert_eq!(items[0]["container"], "App\\Module000\\Service10");
let r = client.call(execute("goto.file", json!({"query": "service10.php"})));
let first = &r["result"]["items"][0];
assert_eq!(first["path"], "src/Module000/Service10.php", "{r}");
assert!(first["matches"].is_array());
let r = client.call(execute("goto.file", json!({"query": 7})));
assert_eq!(r["error"]["code"], "invalid_params", "{r}");
}