//! Symbol index for Lutra (layer L2). //! //! Lists every file of a project and the declarations of its PHP files //! (classes, interfaces, traits, enums, functions, methods, properties, //! constants, enum cases), extracted from tree-sitter outlines //! ([`lutra_syntax::outline`]), and answers fuzzy go-to queries over them. //! //! # Which files //! The walk follows the rules of `lutra-vfs` and `lutra-search`: //! `.gitignore`, `.ignore`, `.git/info/exclude` and the global git excludes //! apply without a git repository; `.git` and every `node_modules` directory //! are skipped; the root's `vendor/` is walked even when an ignore file //! lists it and its files are marked as library code; dot files are listed; //! symbolic links are not followed. Every file is listed for //! [`Index::find_files`]; only `.php` files are parsed, and only those up to //! [`MAX_PARSE_BYTES`]. At most [`MAX_FILES`] files are indexed. //! //! # Building //! [`build`] walks the project, then parses the PHP files on a pool of //! worker threads (one parser each, one file in memory each), project files //! before library files and larger files first within each group. It //! reports [`Update`]s through a callback: batches of indexed files, //! progress, and a final [`Summary`]. [`spawn_build`] runs it on a thread of //! its own and sends the updates over a channel. [`Updater`] re-indexes //! single files and directories later, from disk or from an editor buffer. //! A [`CancelToken`] stops both between files. //! //! # Robustness //! Source files are untrusted: reads are capped, invalid UTF-8 is replaced, //! broken PHP yields the declarations that could be recognised, symbols per //! file are capped ([`MAX_SYMBOLS_PER_FILE`]), and a panic while parsing one //! file (a bug) loses that file's symbols, not the index. mod build; mod query; mod symbol; mod updater; mod walk; use std::sync::Arc; use std::sync::atomic::{AtomicBool, Ordering}; use std::time::Duration; pub use build::{build, spawn_build}; pub use query::{FileHit, Index, MAX_LIMIT, MAX_QUERY_CHARS, SymbolHit}; pub use symbol::{IndexedFile, Symbol, SymbolKind, index_text, is_library, symbols_from_outline}; pub use updater::{Job, Updater}; /// Most files one index holds. pub const MAX_FILES: usize = 200_000; /// Largest PHP file parsed, in bytes (2 MiB). Larger files are listed for /// [`Index::find_files`] without symbols: tree-sitter-php parses about /// 2–3 MB/s, so one such file would hold a worker for a second or more, and /// files this size are generated code rather than declarations to navigate. pub const MAX_PARSE_BYTES: u64 = 2 * 1024 * 1024; /// Most symbols kept from one file; the rest are dropped. pub const MAX_SYMBOLS_PER_FILE: usize = 20_000; /// Most worker threads of one build. pub const MAX_THREADS: usize = 32; /// Files per [`Update::Files`] batch of a build. pub const BATCH_FILES: usize = 256; /// What a build or an [`Updater`] reports. #[derive(Debug, Clone, PartialEq, Eq)] pub enum Update { /// Indexed files that replace any earlier entry with the same path. Files { /// The files. files: Vec, /// From an [`Updater`] (a change after the build started), so newer /// than anything the build reports for the same path. live: bool, }, /// A file or directory is gone: drop `path` and everything below it. Removed { /// Project-relative path with `/` separators. path: String, }, /// Build progress: `done` of `total` files are indexed. Progress { /// Files indexed so far. done: usize, /// Files found by the walk. total: usize, }, /// The build ended; nothing from it follows. Ready(Summary), } /// How a build ended. #[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] pub struct Summary { /// Files indexed. pub files: usize, /// Symbols found in them. pub symbols: usize, /// PHP files parsed. pub parsed: usize, /// Bytes of PHP parsed. pub parsed_bytes: u64, /// The walk found more than [`MAX_FILES`] files; the rest are left out. pub capped: bool, /// The build was cancelled before it finished. pub cancelled: bool, /// Wall time of the walk alone. pub walk: Duration, /// Wall time from start to end. pub elapsed: Duration, } /// Stops a running build or updater. Clones share the flag. #[derive(Debug, Clone, Default)] pub struct CancelToken(Arc); impl CancelToken { /// A token that is not cancelled. pub fn new() -> Self { Self::default() } /// Asks the work to stop at the next file. pub fn cancel(&self) { self.0.store(true, Ordering::Relaxed); } /// Whether [`CancelToken::cancel`] was called. pub fn is_cancelled(&self) -> bool { self.0.load(Ordering::Relaxed) } } /// Worker threads for a build on this machine: one less than the cores, /// so the UI and the engine keep a core, at least one and at most /// [`MAX_THREADS`]. pub fn default_threads() -> usize { std::thread::available_parallelism() .map(|n| n.get()) .unwrap_or(2) .saturating_sub(1) .clamp(1, MAX_THREADS) }