//! Incremental UTF-8 decoding of pty output. /// Turns a stream of bytes, read in arbitrary pieces, into text. /// /// A multibyte character split across two pieces is held back until the /// rest arrives. Invalid sequences become U+FFFD, one per maximal invalid /// subpart, as [`String::from_utf8_lossy`] does. #[derive(Debug, Default)] pub struct Utf8Chunker { /// The start of a character whose remaining bytes have not arrived; /// at most 3 bytes. pending: Vec, } impl Utf8Chunker { /// A chunker with nothing pending. pub fn new() -> Self { Self::default() } /// Decodes the next piece of the stream. /// /// # Returns /// Every complete character so far, possibly empty. An incomplete /// character at the end of `bytes` is kept for the next call. pub fn push(&mut self, bytes: &[u8]) -> String { let mut input = std::mem::take(&mut self.pending); input.extend_from_slice(bytes); let mut out = String::with_capacity(input.len()); let mut rest: &[u8] = &input; loop { match std::str::from_utf8(rest) { Ok(text) => { out.push_str(text); break; } Err(e) => { let (valid, after) = rest.split_at(e.valid_up_to()); out.push_str(&String::from_utf8_lossy(valid)); match e.error_len() { Some(len) => { out.push(char::REPLACEMENT_CHARACTER); rest = after.get(len..).unwrap_or_default(); } None => { self.pending = after.to_vec(); break; } } } } } out } /// Ends the stream: an incomplete character still pending becomes /// U+FFFD. pub fn finish(&mut self) -> String { let pending = std::mem::take(&mut self.pending); String::from_utf8_lossy(&pending).into_owned() } /// Whether bytes of an incomplete character are held back. pub fn has_pending(&self) -> bool { !self.pending.is_empty() } } #[cfg(test)] mod tests { use proptest::prelude::*; use super::Utf8Chunker; #[test] fn joins_a_character_split_across_reads() { let bytes = "aรคโ‚ฌ๐Ÿ˜€".as_bytes(); for split in 0..=bytes.len() { let mut c = Utf8Chunker::new(); let (a, b) = bytes.split_at(split); let text = c.push(a) + &c.push(b) + &c.finish(); assert_eq!(text, "aรคโ‚ฌ๐Ÿ˜€", "split at {split}"); } } #[test] fn one_byte_at_a_time() { let mut c = Utf8Chunker::new(); let mut text = String::new(); for b in "๐Ÿ˜€x".as_bytes() { text.push_str(&c.push(&[*b])); } assert_eq!(text, "๐Ÿ˜€x"); assert!(!c.has_pending()); } #[test] fn invalid_bytes_become_replacement_characters() { let mut c = Utf8Chunker::new(); assert_eq!(c.push(b"a\xffb"), "a\u{fffd}b"); assert_eq!(c.push(b"\xe2\x82"), ""); assert!(c.has_pending()); assert_eq!(c.push(b"x"), "\u{fffd}x"); assert_eq!(c.push(b"\xf0\x9f"), ""); assert_eq!(c.finish(), "\u{fffd}"); } proptest! { #[test] fn matches_lossy_decoding_for_any_split( bytes in proptest::collection::vec(any::(), 0..64), cuts in proptest::collection::vec(0usize..64, 0..6), ) { let mut cuts: Vec = cuts.into_iter().map(|c| c.min(bytes.len())).collect(); cuts.sort_unstable(); let mut c = Utf8Chunker::new(); let mut text = String::new(); let mut start = 0; for cut in cuts.into_iter().chain([bytes.len()]) { text.push_str(&c.push(&bytes[start..cut])); start = cut; } text.push_str(&c.finish()); prop_assert_eq!(text, String::from_utf8_lossy(&bytes).into_owned()); } #[test] fn valid_text_round_trips(text in "\\PC{0,40}", cut in 0usize..200) { let bytes = text.as_bytes(); let cut = cut.min(bytes.len()); let mut c = Utf8Chunker::new(); let out = c.push(&bytes[..cut]) + &c.push(&bytes[cut..]); prop_assert_eq!(out, text); prop_assert!(!c.has_pending()); } } }