crates/index: walks the project with the search/VFS rules (vendor as library), parses .php files from tree-sitter outlines on a worker pool off the engine lock, updates incrementally from disk and buffers through an updater thread, caps at 200,000 files, and answers fuzzy file, class and symbol queries (nucleo-matcher, MPL-2.0 as a separate crate). Engine: index.status, goto.file, goto.class, goto.symbol; events indexProgress and indexReady drained by the pump; the index starts on project.open and is cancelled on close. Tests: unit, incremental, property (random queries and sources), engine and TCP control tests, release timing on a generated 20k-file project (3.9-4.2 s). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
192 lines
5.7 KiB
Rust
192 lines
5.7 KiB
Rust
//! Incremental updates: re-indexing single files and directories after the
|
|
//! build, on a background thread.
|
|
|
|
use std::collections::HashMap;
|
|
use std::path::PathBuf;
|
|
use std::sync::mpsc::{Receiver, Sender, channel};
|
|
|
|
use lutra_syntax::Parser;
|
|
use lutra_text::Buffer;
|
|
|
|
use crate::build::{guarded_index, index_file};
|
|
use crate::walk::walk;
|
|
use crate::{CancelToken, IndexedFile, MAX_FILES, Update};
|
|
|
|
/// Most files one directory rescan reports in one [`Update::Files`].
|
|
const RESCAN_BATCH: usize = 256;
|
|
|
|
/// Work for an [`Updater`].
|
|
#[derive(Debug, Clone)]
|
|
pub enum Job {
|
|
/// Re-index one file.
|
|
File {
|
|
/// Project-relative path with `/` separators.
|
|
path: String,
|
|
/// The text of an open editor buffer to index instead of the file on
|
|
/// disk; `None` reads the disk, and reports [`Update::Removed`] when
|
|
/// the file is gone or now excluded by the project's rules.
|
|
text: Option<Buffer>,
|
|
},
|
|
/// Re-index a directory: drop everything below it, then list it again
|
|
/// from disk.
|
|
Dir {
|
|
/// Project-relative path with `/` separators.
|
|
path: String,
|
|
},
|
|
}
|
|
|
|
impl Job {
|
|
/// The path the job is about.
|
|
fn path(&self) -> &str {
|
|
match self {
|
|
Self::File { path, .. } | Self::Dir { path } => path,
|
|
}
|
|
}
|
|
}
|
|
|
|
/// A background thread that runs [`Job`]s one batch at a time and sends
|
|
/// the results as live [`Update`]s. Jobs queued while it works are
|
|
/// coalesced: only the last job per path runs, so a burst of keystrokes in
|
|
/// one buffer parses it once. The thread ends when the updater is dropped
|
|
/// or `cancel` fires.
|
|
#[derive(Debug)]
|
|
pub struct Updater {
|
|
/// Queues jobs for the thread.
|
|
jobs: Sender<Job>,
|
|
}
|
|
|
|
impl Updater {
|
|
/// Starts the thread for the project at `root`.
|
|
///
|
|
/// # Parameters
|
|
/// - `root`: the project root (absolute).
|
|
/// - `cancel`: stops the thread between jobs.
|
|
/// - `updates`: where results go, usually the build's channel.
|
|
///
|
|
/// # Errors
|
|
/// The thread could not be started.
|
|
pub fn spawn(
|
|
root: PathBuf,
|
|
cancel: CancelToken,
|
|
updates: Sender<Update>,
|
|
) -> std::io::Result<Self> {
|
|
let (jobs, queue) = channel();
|
|
std::thread::Builder::new()
|
|
.name("lutra-index-update".into())
|
|
.spawn(move || run(&root, &cancel, &queue, &updates))?;
|
|
Ok(Self { jobs })
|
|
}
|
|
|
|
/// Queues `job`. Returns `false` if the thread has ended.
|
|
pub fn send(&self, job: Job) -> bool {
|
|
self.jobs.send(job).is_ok()
|
|
}
|
|
}
|
|
|
|
/// The updater thread's loop.
|
|
fn run(
|
|
root: &std::path::Path,
|
|
cancel: &CancelToken,
|
|
queue: &Receiver<Job>,
|
|
updates: &Sender<Update>,
|
|
) {
|
|
let mut parser = Parser::new().ok();
|
|
while let Ok(first) = queue.recv() {
|
|
let mut jobs = vec![first];
|
|
jobs.extend(queue.try_iter());
|
|
for job in coalesce(jobs) {
|
|
if cancel.is_cancelled() {
|
|
return;
|
|
}
|
|
let sent = match job {
|
|
Job::File {
|
|
path,
|
|
text: Some(text),
|
|
} => {
|
|
let library = crate::is_library(&path);
|
|
let file = guarded_index(&mut parser, path.clone(), &text.text()).unwrap_or(
|
|
IndexedFile {
|
|
path,
|
|
library,
|
|
symbols: Vec::new(),
|
|
},
|
|
);
|
|
updates
|
|
.send(Update::Files {
|
|
files: vec![file],
|
|
live: true,
|
|
})
|
|
.is_ok()
|
|
}
|
|
Job::File { path, text: None } => {
|
|
rescan(root, &path, false, cancel, &mut parser, updates)
|
|
}
|
|
Job::Dir { path } => rescan(root, &path, true, cancel, &mut parser, updates),
|
|
};
|
|
if !sent {
|
|
return;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Keeps the last job per path, in the order of those last jobs.
|
|
fn coalesce(jobs: Vec<Job>) -> Vec<Job> {
|
|
let mut last: HashMap<String, usize> = HashMap::new();
|
|
for (i, job) in jobs.iter().enumerate() {
|
|
last.insert(job.path().to_string(), i);
|
|
}
|
|
jobs.into_iter()
|
|
.enumerate()
|
|
.filter(|(i, job)| last.get(job.path()) == Some(i))
|
|
.map(|(_, job)| job)
|
|
.collect()
|
|
}
|
|
|
|
/// Lists `path` from disk with the project's rules and indexes what is
|
|
/// found. A file that is no longer found is reported removed; a directory
|
|
/// is always reported removed first, so files gone from it disappear.
|
|
/// Returns `false` once the receiver is gone.
|
|
fn rescan(
|
|
root: &std::path::Path,
|
|
path: &str,
|
|
dir: bool,
|
|
cancel: &CancelToken,
|
|
parser: &mut Option<Parser>,
|
|
updates: &Sender<Update>,
|
|
) -> bool {
|
|
let walked = walk(root, Some(path), 1, MAX_FILES, cancel);
|
|
if (dir || walked.files.is_empty())
|
|
&& updates
|
|
.send(Update::Removed {
|
|
path: path.to_string(),
|
|
})
|
|
.is_err()
|
|
{
|
|
return false;
|
|
}
|
|
let mut batch = Vec::new();
|
|
for found in &walked.files {
|
|
if cancel.is_cancelled() {
|
|
return true;
|
|
}
|
|
batch.push(index_file(parser, found).0);
|
|
if batch.len() >= RESCAN_BATCH
|
|
&& updates
|
|
.send(Update::Files {
|
|
files: std::mem::take(&mut batch),
|
|
live: true,
|
|
})
|
|
.is_err()
|
|
{
|
|
return false;
|
|
}
|
|
}
|
|
batch.is_empty()
|
|
|| updates
|
|
.send(Update::Files {
|
|
files: batch,
|
|
live: true,
|
|
})
|
|
.is_ok()
|
|
}
|