- crates/terminal: portable-pty sessions (ConPTY on Windows) with reader, writer and waiter threads, UTF-8 chunking, a 1 MiB output buffer with backpressure, a non-blocking input queue, resize, kill and exit codes; default shell pwsh -> powershell -> cmd on Windows, $SHELL or /bin/sh. - Engine: terminal.create/input/resize/close, terminalOutput (capped per terminal and per pump, round-robin) and terminalExited through the pump; terminals close with the project and on app.quit. - UI: Terminal tab in the bottom panel (tabs next to Find), xterm.js with the fit addon, sub-tabs, copy/paste, a focus rule that leaves plain Ctrl keys to the shell, terminal.open (Mod+Shift+T), --lt-terminal-* ANSI tokens in both themes (text colours AA) and a 4.5:1 minimum contrast. - Tests: real-shell crate tests (UTF-8, backpressure), engine and TCP control tests, Vitest with a mocked xterm; live check on port 7936; docs and parity rows. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
137 lines
4.5 KiB
Rust
137 lines
4.5 KiB
Rust
//! Incremental UTF-8 decoding of pty output.
|
|
|
|
/// Turns a stream of bytes, read in arbitrary pieces, into text.
|
|
///
|
|
/// A multibyte character split across two pieces is held back until the
|
|
/// rest arrives. Invalid sequences become U+FFFD, one per maximal invalid
|
|
/// subpart, as [`String::from_utf8_lossy`] does.
|
|
#[derive(Debug, Default)]
|
|
pub struct Utf8Chunker {
|
|
/// The start of a character whose remaining bytes have not arrived;
|
|
/// at most 3 bytes.
|
|
pending: Vec<u8>,
|
|
}
|
|
|
|
impl Utf8Chunker {
|
|
/// A chunker with nothing pending.
|
|
pub fn new() -> Self {
|
|
Self::default()
|
|
}
|
|
|
|
/// Decodes the next piece of the stream.
|
|
///
|
|
/// # Returns
|
|
/// Every complete character so far, possibly empty. An incomplete
|
|
/// character at the end of `bytes` is kept for the next call.
|
|
pub fn push(&mut self, bytes: &[u8]) -> String {
|
|
let mut input = std::mem::take(&mut self.pending);
|
|
input.extend_from_slice(bytes);
|
|
let mut out = String::with_capacity(input.len());
|
|
let mut rest: &[u8] = &input;
|
|
loop {
|
|
match std::str::from_utf8(rest) {
|
|
Ok(text) => {
|
|
out.push_str(text);
|
|
break;
|
|
}
|
|
Err(e) => {
|
|
let (valid, after) = rest.split_at(e.valid_up_to());
|
|
out.push_str(&String::from_utf8_lossy(valid));
|
|
match e.error_len() {
|
|
Some(len) => {
|
|
out.push(char::REPLACEMENT_CHARACTER);
|
|
rest = after.get(len..).unwrap_or_default();
|
|
}
|
|
None => {
|
|
self.pending = after.to_vec();
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
out
|
|
}
|
|
|
|
/// Ends the stream: an incomplete character still pending becomes
|
|
/// U+FFFD.
|
|
pub fn finish(&mut self) -> String {
|
|
let pending = std::mem::take(&mut self.pending);
|
|
String::from_utf8_lossy(&pending).into_owned()
|
|
}
|
|
|
|
/// Whether bytes of an incomplete character are held back.
|
|
pub fn has_pending(&self) -> bool {
|
|
!self.pending.is_empty()
|
|
}
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use proptest::prelude::*;
|
|
|
|
use super::Utf8Chunker;
|
|
|
|
#[test]
|
|
fn joins_a_character_split_across_reads() {
|
|
let bytes = "aä€😀".as_bytes();
|
|
for split in 0..=bytes.len() {
|
|
let mut c = Utf8Chunker::new();
|
|
let (a, b) = bytes.split_at(split);
|
|
let text = c.push(a) + &c.push(b) + &c.finish();
|
|
assert_eq!(text, "aä€😀", "split at {split}");
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn one_byte_at_a_time() {
|
|
let mut c = Utf8Chunker::new();
|
|
let mut text = String::new();
|
|
for b in "😀x".as_bytes() {
|
|
text.push_str(&c.push(&[*b]));
|
|
}
|
|
assert_eq!(text, "😀x");
|
|
assert!(!c.has_pending());
|
|
}
|
|
|
|
#[test]
|
|
fn invalid_bytes_become_replacement_characters() {
|
|
let mut c = Utf8Chunker::new();
|
|
assert_eq!(c.push(b"a\xffb"), "a\u{fffd}b");
|
|
assert_eq!(c.push(b"\xe2\x82"), "");
|
|
assert!(c.has_pending());
|
|
assert_eq!(c.push(b"x"), "\u{fffd}x");
|
|
assert_eq!(c.push(b"\xf0\x9f"), "");
|
|
assert_eq!(c.finish(), "\u{fffd}");
|
|
}
|
|
|
|
proptest! {
|
|
#[test]
|
|
fn matches_lossy_decoding_for_any_split(
|
|
bytes in proptest::collection::vec(any::<u8>(), 0..64),
|
|
cuts in proptest::collection::vec(0usize..64, 0..6),
|
|
) {
|
|
let mut cuts: Vec<usize> = cuts.into_iter().map(|c| c.min(bytes.len())).collect();
|
|
cuts.sort_unstable();
|
|
let mut c = Utf8Chunker::new();
|
|
let mut text = String::new();
|
|
let mut start = 0;
|
|
for cut in cuts.into_iter().chain([bytes.len()]) {
|
|
text.push_str(&c.push(&bytes[start..cut]));
|
|
start = cut;
|
|
}
|
|
text.push_str(&c.finish());
|
|
prop_assert_eq!(text, String::from_utf8_lossy(&bytes).into_owned());
|
|
}
|
|
|
|
#[test]
|
|
fn valid_text_round_trips(text in "\\PC{0,40}", cut in 0usize..200) {
|
|
let bytes = text.as_bytes();
|
|
let cut = cut.min(bytes.len());
|
|
let mut c = Utf8Chunker::new();
|
|
let out = c.push(&bytes[..cut]) + &c.push(&bytes[cut..]);
|
|
prop_assert_eq!(out, text);
|
|
prop_assert!(!c.has_pending());
|
|
}
|
|
}
|
|
}
|