Lutra/crates/index/src/updater.rs
danny 053441d066 M2.5: symbol index and go-to commands
crates/index: walks the project with the search/VFS rules (vendor as
library), parses .php files from tree-sitter outlines on a worker pool
off the engine lock, updates incrementally from disk and buffers through
an updater thread, caps at 200,000 files, and answers fuzzy file, class
and symbol queries (nucleo-matcher, MPL-2.0 as a separate crate).

Engine: index.status, goto.file, goto.class, goto.symbol; events
indexProgress and indexReady drained by the pump; the index starts on
project.open and is cancelled on close. Tests: unit, incremental,
property (random queries and sources), engine and TCP control tests,
release timing on a generated 20k-file project (3.9-4.2 s).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-17 12:53:33 +02:00

192 lines
5.7 KiB
Rust

//! Incremental updates: re-indexing single files and directories after the
//! build, on a background thread.
use std::collections::HashMap;
use std::path::PathBuf;
use std::sync::mpsc::{Receiver, Sender, channel};
use lutra_syntax::Parser;
use lutra_text::Buffer;
use crate::build::{guarded_index, index_file};
use crate::walk::walk;
use crate::{CancelToken, IndexedFile, MAX_FILES, Update};
/// Most files one directory rescan reports in one [`Update::Files`].
const RESCAN_BATCH: usize = 256;
/// Work for an [`Updater`].
#[derive(Debug, Clone)]
pub enum Job {
/// Re-index one file.
File {
/// Project-relative path with `/` separators.
path: String,
/// The text of an open editor buffer to index instead of the file on
/// disk; `None` reads the disk, and reports [`Update::Removed`] when
/// the file is gone or now excluded by the project's rules.
text: Option<Buffer>,
},
/// Re-index a directory: drop everything below it, then list it again
/// from disk.
Dir {
/// Project-relative path with `/` separators.
path: String,
},
}
impl Job {
/// The path the job is about.
fn path(&self) -> &str {
match self {
Self::File { path, .. } | Self::Dir { path } => path,
}
}
}
/// A background thread that runs [`Job`]s one batch at a time and sends
/// the results as live [`Update`]s. Jobs queued while it works are
/// coalesced: only the last job per path runs, so a burst of keystrokes in
/// one buffer parses it once. The thread ends when the updater is dropped
/// or `cancel` fires.
#[derive(Debug)]
pub struct Updater {
/// Queues jobs for the thread.
jobs: Sender<Job>,
}
impl Updater {
/// Starts the thread for the project at `root`.
///
/// # Parameters
/// - `root`: the project root (absolute).
/// - `cancel`: stops the thread between jobs.
/// - `updates`: where results go, usually the build's channel.
///
/// # Errors
/// The thread could not be started.
pub fn spawn(
root: PathBuf,
cancel: CancelToken,
updates: Sender<Update>,
) -> std::io::Result<Self> {
let (jobs, queue) = channel();
std::thread::Builder::new()
.name("lutra-index-update".into())
.spawn(move || run(&root, &cancel, &queue, &updates))?;
Ok(Self { jobs })
}
/// Queues `job`. Returns `false` if the thread has ended.
pub fn send(&self, job: Job) -> bool {
self.jobs.send(job).is_ok()
}
}
/// The updater thread's loop.
fn run(
root: &std::path::Path,
cancel: &CancelToken,
queue: &Receiver<Job>,
updates: &Sender<Update>,
) {
let mut parser = Parser::new().ok();
while let Ok(first) = queue.recv() {
let mut jobs = vec![first];
jobs.extend(queue.try_iter());
for job in coalesce(jobs) {
if cancel.is_cancelled() {
return;
}
let sent = match job {
Job::File {
path,
text: Some(text),
} => {
let library = crate::is_library(&path);
let file = guarded_index(&mut parser, path.clone(), &text.text()).unwrap_or(
IndexedFile {
path,
library,
symbols: Vec::new(),
},
);
updates
.send(Update::Files {
files: vec![file],
live: true,
})
.is_ok()
}
Job::File { path, text: None } => {
rescan(root, &path, false, cancel, &mut parser, updates)
}
Job::Dir { path } => rescan(root, &path, true, cancel, &mut parser, updates),
};
if !sent {
return;
}
}
}
}
/// Keeps the last job per path, in the order of those last jobs.
fn coalesce(jobs: Vec<Job>) -> Vec<Job> {
let mut last: HashMap<String, usize> = HashMap::new();
for (i, job) in jobs.iter().enumerate() {
last.insert(job.path().to_string(), i);
}
jobs.into_iter()
.enumerate()
.filter(|(i, job)| last.get(job.path()) == Some(i))
.map(|(_, job)| job)
.collect()
}
/// Lists `path` from disk with the project's rules and indexes what is
/// found. A file that is no longer found is reported removed; a directory
/// is always reported removed first, so files gone from it disappear.
/// Returns `false` once the receiver is gone.
fn rescan(
root: &std::path::Path,
path: &str,
dir: bool,
cancel: &CancelToken,
parser: &mut Option<Parser>,
updates: &Sender<Update>,
) -> bool {
let walked = walk(root, Some(path), 1, MAX_FILES, cancel);
if (dir || walked.files.is_empty())
&& updates
.send(Update::Removed {
path: path.to_string(),
})
.is_err()
{
return false;
}
let mut batch = Vec::new();
for found in &walked.files {
if cancel.is_cancelled() {
return true;
}
batch.push(index_file(parser, found).0);
if batch.len() >= RESCAN_BATCH
&& updates
.send(Update::Files {
files: std::mem::take(&mut batch),
live: true,
})
.is_err()
{
return false;
}
}
batch.is_empty()
|| updates
.send(Update::Files {
files: batch,
live: true,
})
.is_ok()
}