M2.5: symbol index and go-to commands
crates/index: walks the project with the search/VFS rules (vendor as library), parses .php files from tree-sitter outlines on a worker pool off the engine lock, updates incrementally from disk and buffers through an updater thread, caps at 200,000 files, and answers fuzzy file, class and symbol queries (nucleo-matcher, MPL-2.0 as a separate crate). Engine: index.status, goto.file, goto.class, goto.symbol; events indexProgress and indexReady drained by the pump; the index starts on project.open and is cancelled on close. Tests: unit, incremental, property (random queries and sources), engine and TCP control tests, release timing on a generated 20k-file project (3.9-4.2 s). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
f13a49064d
commit
053441d066
26 changed files with 3455 additions and 9 deletions
192
crates/index/src/updater.rs
Normal file
192
crates/index/src/updater.rs
Normal file
|
|
@ -0,0 +1,192 @@
|
|||
//! Incremental updates: re-indexing single files and directories after the
|
||||
//! build, on a background thread.
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::path::PathBuf;
|
||||
use std::sync::mpsc::{Receiver, Sender, channel};
|
||||
|
||||
use lutra_syntax::Parser;
|
||||
use lutra_text::Buffer;
|
||||
|
||||
use crate::build::{guarded_index, index_file};
|
||||
use crate::walk::walk;
|
||||
use crate::{CancelToken, IndexedFile, MAX_FILES, Update};
|
||||
|
||||
/// Most files one directory rescan reports in one [`Update::Files`].
|
||||
const RESCAN_BATCH: usize = 256;
|
||||
|
||||
/// Work for an [`Updater`].
|
||||
#[derive(Debug, Clone)]
|
||||
pub enum Job {
|
||||
/// Re-index one file.
|
||||
File {
|
||||
/// Project-relative path with `/` separators.
|
||||
path: String,
|
||||
/// The text of an open editor buffer to index instead of the file on
|
||||
/// disk; `None` reads the disk, and reports [`Update::Removed`] when
|
||||
/// the file is gone or now excluded by the project's rules.
|
||||
text: Option<Buffer>,
|
||||
},
|
||||
/// Re-index a directory: drop everything below it, then list it again
|
||||
/// from disk.
|
||||
Dir {
|
||||
/// Project-relative path with `/` separators.
|
||||
path: String,
|
||||
},
|
||||
}
|
||||
|
||||
impl Job {
|
||||
/// The path the job is about.
|
||||
fn path(&self) -> &str {
|
||||
match self {
|
||||
Self::File { path, .. } | Self::Dir { path } => path,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// A background thread that runs [`Job`]s one batch at a time and sends
|
||||
/// the results as live [`Update`]s. Jobs queued while it works are
|
||||
/// coalesced: only the last job per path runs, so a burst of keystrokes in
|
||||
/// one buffer parses it once. The thread ends when the updater is dropped
|
||||
/// or `cancel` fires.
|
||||
#[derive(Debug)]
|
||||
pub struct Updater {
|
||||
/// Queues jobs for the thread.
|
||||
jobs: Sender<Job>,
|
||||
}
|
||||
|
||||
impl Updater {
|
||||
/// Starts the thread for the project at `root`.
|
||||
///
|
||||
/// # Parameters
|
||||
/// - `root`: the project root (absolute).
|
||||
/// - `cancel`: stops the thread between jobs.
|
||||
/// - `updates`: where results go, usually the build's channel.
|
||||
///
|
||||
/// # Errors
|
||||
/// The thread could not be started.
|
||||
pub fn spawn(
|
||||
root: PathBuf,
|
||||
cancel: CancelToken,
|
||||
updates: Sender<Update>,
|
||||
) -> std::io::Result<Self> {
|
||||
let (jobs, queue) = channel();
|
||||
std::thread::Builder::new()
|
||||
.name("lutra-index-update".into())
|
||||
.spawn(move || run(&root, &cancel, &queue, &updates))?;
|
||||
Ok(Self { jobs })
|
||||
}
|
||||
|
||||
/// Queues `job`. Returns `false` if the thread has ended.
|
||||
pub fn send(&self, job: Job) -> bool {
|
||||
self.jobs.send(job).is_ok()
|
||||
}
|
||||
}
|
||||
|
||||
/// The updater thread's loop.
|
||||
fn run(
|
||||
root: &std::path::Path,
|
||||
cancel: &CancelToken,
|
||||
queue: &Receiver<Job>,
|
||||
updates: &Sender<Update>,
|
||||
) {
|
||||
let mut parser = Parser::new().ok();
|
||||
while let Ok(first) = queue.recv() {
|
||||
let mut jobs = vec![first];
|
||||
jobs.extend(queue.try_iter());
|
||||
for job in coalesce(jobs) {
|
||||
if cancel.is_cancelled() {
|
||||
return;
|
||||
}
|
||||
let sent = match job {
|
||||
Job::File {
|
||||
path,
|
||||
text: Some(text),
|
||||
} => {
|
||||
let library = crate::is_library(&path);
|
||||
let file = guarded_index(&mut parser, path.clone(), &text.text()).unwrap_or(
|
||||
IndexedFile {
|
||||
path,
|
||||
library,
|
||||
symbols: Vec::new(),
|
||||
},
|
||||
);
|
||||
updates
|
||||
.send(Update::Files {
|
||||
files: vec![file],
|
||||
live: true,
|
||||
})
|
||||
.is_ok()
|
||||
}
|
||||
Job::File { path, text: None } => {
|
||||
rescan(root, &path, false, cancel, &mut parser, updates)
|
||||
}
|
||||
Job::Dir { path } => rescan(root, &path, true, cancel, &mut parser, updates),
|
||||
};
|
||||
if !sent {
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Keeps the last job per path, in the order of those last jobs.
|
||||
fn coalesce(jobs: Vec<Job>) -> Vec<Job> {
|
||||
let mut last: HashMap<String, usize> = HashMap::new();
|
||||
for (i, job) in jobs.iter().enumerate() {
|
||||
last.insert(job.path().to_string(), i);
|
||||
}
|
||||
jobs.into_iter()
|
||||
.enumerate()
|
||||
.filter(|(i, job)| last.get(job.path()) == Some(i))
|
||||
.map(|(_, job)| job)
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Lists `path` from disk with the project's rules and indexes what is
|
||||
/// found. A file that is no longer found is reported removed; a directory
|
||||
/// is always reported removed first, so files gone from it disappear.
|
||||
/// Returns `false` once the receiver is gone.
|
||||
fn rescan(
|
||||
root: &std::path::Path,
|
||||
path: &str,
|
||||
dir: bool,
|
||||
cancel: &CancelToken,
|
||||
parser: &mut Option<Parser>,
|
||||
updates: &Sender<Update>,
|
||||
) -> bool {
|
||||
let walked = walk(root, Some(path), 1, MAX_FILES, cancel);
|
||||
if (dir || walked.files.is_empty())
|
||||
&& updates
|
||||
.send(Update::Removed {
|
||||
path: path.to_string(),
|
||||
})
|
||||
.is_err()
|
||||
{
|
||||
return false;
|
||||
}
|
||||
let mut batch = Vec::new();
|
||||
for found in &walked.files {
|
||||
if cancel.is_cancelled() {
|
||||
return true;
|
||||
}
|
||||
batch.push(index_file(parser, found).0);
|
||||
if batch.len() >= RESCAN_BATCH
|
||||
&& updates
|
||||
.send(Update::Files {
|
||||
files: std::mem::take(&mut batch),
|
||||
live: true,
|
||||
})
|
||||
.is_err()
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
batch.is_empty()
|
||||
|| updates
|
||||
.send(Update::Files {
|
||||
files: batch,
|
||||
live: true,
|
||||
})
|
||||
.is_ok()
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue