//! `git.lineChanges` (M4.4): which lines of an open buffer were added, //! modified or deleted against the file's version in the Git index, for //! the editor's gutter markers. //! //! **Off the lock.** Under the lock the command only clones the buffer's //! rope (cheap: ropes share their chunks) and takes the cached repository //! and index text. A `lutra-git-lines` thread then finds git and the //! repository if needed, reads the index text with `git show :0:` //! if it is not cached, and diffs the lines with `similar` (Myers, with a //! deadline). The finish, under the lock again, caches what the thread //! found and returns the ranges. //! //! **Cache.** The index text of each path is cached (normalised to `\n` //! line breaks), and so is the repository lookup. Every `gitChanged` //! clears the index texts ([`observe`], called by [`State::emit`]); a //! project change clears everything. A thread that started before a //! `gitChanged` does not store what it read. "No repository" and "no //! usable git" are cached for [`NEGATIVE_TTL`] only, since no `gitChanged` //! arrives without a repository. //! //! **Lines.** Both texts are split at `\n`, `\r\n` and a lone `\r` (the //! index holds the blob as stored, usually LF, while a buffer may be //! CRLF), with no empty line after a final break. Lines are 0-based //! (CodeMirror's line number minus one). use std::collections::HashMap; use std::path::{Path, PathBuf}; use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::mpsc; use std::sync::{Arc, Mutex, PoisonError}; use std::time::{Duration, Instant}; use lutra_text::Buffer; use lutra_vcs::{Git, Repo, Rev}; use serde::{Deserialize, Serialize}; use serde_json::{Value, json}; use similar::DiffOp; use similar::algorithms::{Capture, Replace, myers}; use super::params::parse; use crate::buffers::BufferId; use crate::workspace::path_string; use crate::{Command, CommandSpec, EngineError, Event, Outcome, Pending, Registry, State}; /// How long "no repository" or "no usable git" is believed. pub const NEGATIVE_TTL: Duration = Duration::from_secs(10); /// Longest the line diff may take before it settles for a coarser result. const DIFF_DEADLINE: Duration = Duration::from_secs(1); /// Most ranges returned; a diff with more is cut. pub const MAX_RANGES: usize = 10_000; /// Most index texts cached. const MAX_CACHED: usize = 256; /// Most bytes of index text cached; the cache is cleared beyond it. const MAX_CACHED_BYTES: usize = 64 * 1024 * 1024; /// Most calls running at once; more are refused. const MAX_RUNNING: usize = 8; /// Longest wait for a call: git detection, opening and `show` each have /// their own 30 s timeout in `lutra-vcs`; the runner kills git first. const WAIT: Duration = Duration::from_secs(100); /// Registers `git.lineChanges`. /// /// # Errors /// [`EngineError::Internal`] if the id is taken. pub fn register(registry: &mut Registry) -> Result<(), EngineError> { registry.register(Box::new(LineChanges)) } /// The result of looking for the project's repository. #[derive(Debug, Clone)] struct Lookup { /// The project root it was made for. project: PathBuf, /// The `git.executable` value it was made with. setting: String, /// The repository; `None` when there is none or git is unusable. repo: Option>, /// When it was made. at: Instant, } impl Lookup { /// Whether the lookup still applies to `project` and `setting` at `now`. fn applies(&self, project: &Path, setting: &str, now: Instant) -> bool { self.project == project && self.setting == setting && (self.repo.is_some() || now.saturating_duration_since(self.at) < NEGATIVE_TTL) } } /// An index text; `None` when the path is not in the index, or the blob is /// binary or too large. type IndexText = Option>; /// Cached state of `git.lineChanges`. #[derive(Debug, Default)] pub struct GitLines { /// The repository lookup. lookup: Option, /// Index texts by project-relative path. index: HashMap, /// Bytes held by `index`. bytes: usize, /// Incremented whenever the cache is cleared, so a thread that started /// before does not store stale text. generation: u64, /// Calls running now. running: Arc, } impl GitLines { /// Drops the cached index texts. fn clear_index(&mut self) { self.index.clear(); self.bytes = 0; self.generation = self.generation.wrapping_add(1); } /// Stores the index text of `path`, clearing the cache first when it /// would grow past its caps. fn store(&mut self, path: String, text: IndexText) { let size = text.as_ref().map_or(0, |t| t.len()); if self.index.len() >= MAX_CACHED || self.bytes.saturating_add(size) > MAX_CACHED_BYTES { self.index.clear(); self.bytes = 0; } if size > MAX_CACHED_BYTES { return; } if let Some(old) = self.index.insert(path, text) { self.bytes = self .bytes .saturating_sub(old.as_ref().map_or(0, |t| t.len())); } self.bytes = self.bytes.saturating_add(size); } } /// Clears the cache for events that make it stale: `gitChanged` drops the /// index texts, a project change everything. Called for every event by /// [`State::emit`]. pub fn observe(lines: &mut GitLines, event: &Event) { match event { Event::GitChanged(_) => lines.clear_index(), Event::ProjectOpened { .. } | Event::ProjectClosed => { lines.clear_index(); lines.lookup = None; } _ => {} } } /// What a line changed: `added` lines are new, `modified` lines replace /// others, and `deleted` marks where lines were removed. #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] #[serde(rename_all = "camelCase")] pub enum ChangeKind { /// New lines. Added, /// Changed lines. Modified, /// Lines removed before `start_line`. Deleted, } /// One changed range of buffer lines. #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] #[serde(rename_all = "camelCase")] pub struct LineRange { /// What changed. pub kind: ChangeKind, /// The first line (0-based). pub start_line: usize, /// The line after the last one; equal to `start_line` for `deleted`, /// where the removed lines were before `start_line` (which may be the /// line count when they were at the end). pub end_line: usize, } /// Splits `text` into lines at `\n`, `\r\n` and a lone `\r`, without an /// empty line after a final break. fn split_lines(text: &str) -> Vec<&str> { let mut lines = Vec::new(); let mut rest = text; while !rest.is_empty() { match rest.find(['\n', '\r']) { Some(at) => { lines.push(rest.get(..at).unwrap_or_default()); let skip = if rest.get(at..at + 2) == Some("\r\n") { 2 } else { 1 }; rest = rest.get(at + skip..).unwrap_or_default(); } None => { lines.push(rest); rest = ""; } } } lines } /// The changed ranges of `new` against `old`, at most [`MAX_RANGES`]; /// the diff gives up refining after `deadline`. /// /// Myers' diff with adjacent deletes and inserts merged into replacements. /// `similar`'s `Compact` step is left out on purpose: in 2.7.0 it can move /// a delete to a wrong position in the new text (found by the property /// test below). pub fn line_changes(old: &str, new: &str, deadline: Option) -> Vec { let old = split_lines(old); let new = split_lines(new); let mut hook = Replace::new(Capture::new()); if myers::diff_deadline(&mut hook, &old, 0..old.len(), &new, 0..new.len(), deadline).is_err() { return Vec::new(); } let ops = hook.into_inner().into_ops(); ops.iter() .filter_map(|op| match *op { DiffOp::Equal { .. } => None, DiffOp::Insert { new_index, new_len, .. } => Some(LineRange { kind: ChangeKind::Added, start_line: new_index, end_line: new_index + new_len, }), DiffOp::Delete { new_index, .. } => Some(LineRange { kind: ChangeKind::Deleted, start_line: new_index, end_line: new_index, }), DiffOp::Replace { new_index, new_len, .. } => Some(LineRange { kind: ChangeKind::Modified, start_line: new_index, end_line: new_index + new_len, }), }) .take(MAX_RANGES) .collect() } /// What the thread hands back to the finish. struct Reply { /// The repository lookup, when this call made one. lookup: Option>>, /// The index text, when this call read it. index: Option, /// The value or error. value: Result, } /// What the thread needs, taken under the lock. struct Job { /// The buffer's text. buffer: Buffer, /// Its version. version: u64, /// Its project-relative path. path: String, /// The project root. project: PathBuf, /// The `git.executable` setting. setting: String, /// The repository lookup, if one applies. lookup: Option>>, /// The cached index text, if any. index: Option, } /// The value for a buffer without index text to compare with. fn unchanged(version: u64) -> Value { json!({"ranges": [], "version": version}) } /// The thread's work: find the repository and read the index text if they /// are not cached, then diff. fn work(job: Job) -> Reply { let mut reply = Reply { lookup: None, index: None, value: Ok(unchanged(job.version)), }; let repo = match job.lookup { Some(repo) => repo, None => { let repo = Git::detect(&job.setting) .ok() .and_then(|git| git.open(&job.project).ok().flatten()) .map(Arc::new); reply.lookup = Some(repo.clone()); repo } }; let Some(repo) = repo else { return reply; }; let index = match job.index { Some(index) => index, None => match repo.show(&job.path, Rev::Index) { Ok(contents) => { let text: IndexText = contents.text.map(Arc::from); reply.index = Some(text.clone()); text } Err(e) => { reply.value = Err(EngineError::Failed(e.to_string())); return reply; } }, }; let Some(index) = index else { return reply; }; let text = job.buffer.text(); let ranges = line_changes(&index, &text, Instant::now().checked_add(DIFF_DEADLINE)); reply.value = Ok(json!({"ranges": ranges, "version": job.version})); reply } /// Params of `git.lineChanges`. #[derive(Debug, Deserialize)] #[serde(deny_unknown_fields)] struct LineChangesParams { /// Buffer id. id: BufferId, } /// `git.lineChanges`. struct LineChanges; impl Command for LineChanges { fn spec(&self) -> &CommandSpec { &CommandSpec { id: "git.lineChanges", label: "Git: Line Changes", menu_path: None, default_shortcut: None, params: "{ id: buffer id }; returns { ranges: [{ kind: \"added\"|\"modified\"|\"deleted\", startLine, endLine }], version } against the file in the Git index (lines 0-based, endLine exclusive; a deleted range has startLine = endLine, the removed lines were before it); no ranges when the project has no repository, git is unusable or the file is not in the index (untracked, binary, too large); the buffer version the ranges are for", } } fn enabled(&self, state: &State) -> bool { state.has_project() } fn run(&self, state: &mut State, params: Value) -> Result { let command = self.spec().id; let p: LineChangesParams = parse(command, params)?; let project = state .project .as_ref() .ok_or_else(|| EngineError::Disabled(command.to_string()))? .project .root() .to_path_buf(); let open = state .buffers .get(p.id) .ok_or_else(|| EngineError::Failed(format!("{command}: no buffer {}", p.id)))?; let path = path_string(&open.path); let buffer = open.buffer.clone(); let version = open.version; let setting = state .settings .get(super::git::EXECUTABLE_SETTING) .and_then(|v| v.as_str().map(str::to_string)) .unwrap_or_default(); let lines = &mut state.git_lines; let now = Instant::now(); if lines .lookup .as_ref() .is_some_and(|l| !l.applies(&project, &setting, now)) { lines.lookup = None; lines.clear_index(); } let lookup = lines.lookup.as_ref().map(|l| l.repo.clone()); let index = lines.index.get(&path).cloned(); let generation = lines.generation; let running = Arc::clone(&lines.running); if running.fetch_add(1, Ordering::SeqCst) >= MAX_RUNNING { running.fetch_sub(1, Ordering::SeqCst); return Err(EngineError::Failed( "too many line change queries are running".into(), )); } let job = Job { buffer, version, path: path.clone(), project: project.clone(), setting: setting.clone(), lookup, index, }; let slot: Arc>> = Arc::new(Mutex::new(None)); let (tx, rx) = mpsc::channel(); let filled = Arc::clone(&slot); let spawned = std::thread::Builder::new() .name("lutra-git-lines".into()) .spawn(move || { let reply = work(job); *filled.lock().unwrap_or_else(PoisonError::into_inner) = Some(reply); running.fetch_sub(1, Ordering::SeqCst); let _ = tx.send(Ok(Value::Null)); }); if let Err(e) = spawned { state.git_lines.running.fetch_sub(1, Ordering::SeqCst); return Err(EngineError::Failed(format!("cannot start git: {e}"))); } Ok(Outcome::pending(Pending::new(rx, WAIT, move |state, _| { let reply = slot .lock() .unwrap_or_else(PoisonError::into_inner) .take() .ok_or_else(|| EngineError::Failed("git did not answer in time".into()))?; let current = state .project .as_ref() .map(|w| w.project.root().to_path_buf()); let lines = &mut state.git_lines; if current.as_deref() == Some(project.as_path()) && lines.generation == generation { if let Some(repo) = reply.lookup { lines.lookup = Some(Lookup { project, setting, repo, at: Instant::now(), }); } if let Some(text) = reply.index { lines.store(path, text); } } reply.value }))) } } /// The ids of the commands this module registers, checked against the /// registry by the engine's listing test. Add a new command's id here. #[cfg(test)] pub(crate) const COMMAND_IDS: &[&str] = &["git.lineChanges"]; #[cfg(test)] mod tests { use super::*; fn range(kind: ChangeKind, start_line: usize, end_line: usize) -> LineRange { LineRange { kind, start_line, end_line, } } #[test] fn splits_every_line_break_without_a_trailing_empty_line() { assert_eq!(split_lines(""), Vec::<&str>::new()); assert_eq!(split_lines("a"), vec!["a"]); assert_eq!(split_lines("a\n"), vec!["a"]); assert_eq!(split_lines("a\r\nb\rc\n\n"), vec!["a", "b", "c", ""]); } #[test] fn finds_added_modified_and_deleted_lines() { let old = "a\nb\nc\nd\ne\n"; assert_eq!(line_changes(old, old, None), vec![]); assert_eq!( line_changes(old, "a\nb\nx\ny\nc\nd\ne\n", None), vec![range(ChangeKind::Added, 2, 4)] ); assert_eq!( line_changes(old, "a\nB\nc\nd\ne\n", None), vec![range(ChangeKind::Modified, 1, 2)] ); assert_eq!( line_changes(old, "a\nd\ne\n", None), vec![range(ChangeKind::Deleted, 1, 1)] ); assert_eq!( line_changes(old, "a\nb\nc\n", None), vec![range(ChangeKind::Deleted, 3, 3)] ); assert_eq!( line_changes(old, "X\nb\nc\nd\n", None), vec![ range(ChangeKind::Modified, 0, 1), range(ChangeKind::Deleted, 4, 4) ] ); } #[test] fn ranges_are_in_order_where_compaction_would_misplace_a_delete() { let old = "aa\n\nb\ra\n\n\nb\n\nb\n\r"; let new = "\n\na\n\ra\na\ra\r\r\ra\n\na\r\ra"; let ranges = line_changes(old, new, None); assert!(!ranges.is_empty()); assert!( ranges.windows(2).all(|w| w[0].end_line <= w[1].start_line), "{ranges:?}" ); } #[test] fn line_endings_do_not_count_as_changes() { assert_eq!(line_changes("a\nb\n", "a\r\nb\r\n", None), vec![]); assert_eq!(line_changes("a\nb", "a\nb\n", None), vec![]); assert_eq!( line_changes("a\r\nb\r\n", "a\r\nb\r\nc\r\n", None), vec![range(ChangeKind::Added, 2, 3)] ); } #[test] fn an_empty_index_file_makes_every_line_added() { assert_eq!( line_changes("", "a\nb\n", None), vec![range(ChangeKind::Added, 0, 2)] ); assert_eq!( line_changes("a\nb\n", "", None), vec![range(ChangeKind::Deleted, 0, 0)] ); } #[test] fn the_cache_respects_its_caps_and_generation() { let mut lines = GitLines::default(); lines.store("a".into(), Some(Arc::from("abc"))); lines.store("a".into(), Some(Arc::from("abcd"))); lines.store("b".into(), None); assert_eq!(lines.bytes, 4); let before = lines.generation; observe( &mut lines, &Event::GitChanged(crate::cmd::git::GitChanged { available: true, repo: None, files_changed: true, }), ); assert!(lines.index.is_empty()); assert_eq!(lines.bytes, 0); assert_ne!(lines.generation, before); for i in 0..=MAX_CACHED { lines.store(i.to_string(), None); } assert!(lines.index.len() <= MAX_CACHED); } proptest::proptest! { #[test] fn ranges_stay_inside_the_buffer( old in "[ab\r\n]{0,40}", new in "[ab\r\n]{0,40}", ) { let count = split_lines(&new).len(); let ranges = line_changes(&old, &new, None); let mut last = 0; for r in &ranges { proptest::prop_assert!(r.start_line <= r.end_line); proptest::prop_assert!(r.end_line <= count); proptest::prop_assert!(r.start_line >= last); if r.kind == ChangeKind::Deleted { proptest::prop_assert_eq!(r.start_line, r.end_line); } else { proptest::prop_assert!(r.start_line < r.end_line); } last = r.end_line; } if split_lines(&old) == split_lines(&new) { proptest::prop_assert!(ranges.is_empty()); } } } }