Unnamed repository; edit this file 'description' to name the repository.
Diffstat (limited to 'helix-core/src/graphemes.rs')
| -rw-r--r-- | helix-core/src/graphemes.rs | 334 |
1 files changed, 168 insertions, 166 deletions
diff --git a/helix-core/src/graphemes.rs b/helix-core/src/graphemes.rs index 4cbb5746..675f5750 100644 --- a/helix-core/src/graphemes.rs +++ b/helix-core/src/graphemes.rs @@ -1,97 +1,11 @@ //! Utility functions to traverse the unicode graphemes of a `Rope`'s text contents. //! //! Based on <https://github.com/cessen/led/blob/c4fa72405f510b7fd16052f90a598c429b3104a6/src/graphemes.rs> -use ropey::{str_utils::byte_to_char_idx, RopeSlice}; +use ropey::{iter::Chunks, str_utils::byte_to_char_idx, RopeSlice}; use unicode_segmentation::{GraphemeCursor, GraphemeIncomplete}; use unicode_width::UnicodeWidthStr; -use std::borrow::Cow; -use std::fmt::{self, Debug, Display}; -use std::marker::PhantomData; -use std::ops::Deref; -use std::ptr::NonNull; -use std::{slice, str}; - -use crate::chars::{char_is_whitespace, char_is_word}; -use crate::LineEnding; - -#[inline] -pub fn tab_width_at(visual_x: usize, tab_width: u16) -> usize { - tab_width as usize - (visual_x % tab_width as usize) -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum Grapheme<'a> { - Newline, - Tab { width: usize }, - Other { g: GraphemeStr<'a> }, -} - -impl<'a> Grapheme<'a> { - pub fn new_decoration(g: &'static str) -> Grapheme<'a> { - assert_ne!(g, "\t"); - Grapheme::new(g.into(), 0, 0) - } - - pub fn new(g: GraphemeStr<'a>, visual_x: usize, tab_width: u16) -> Grapheme<'a> { - match g { - g if g == "\t" => Grapheme::Tab { - width: tab_width_at(visual_x, tab_width), - }, - _ if LineEnding::from_str(&g).is_some() => Grapheme::Newline, - _ => Grapheme::Other { g }, - } - } - - pub fn change_position(&mut self, visual_x: usize, tab_width: u16) { - if let Grapheme::Tab { width } = self { - *width = tab_width_at(visual_x, tab_width) - } - } - - /// Returns the a visual width of this grapheme, - #[inline] - pub fn width(&self) -> usize { - match *self { - // width is not cached because we are dealing with - // ASCII almost all the time which already has a fastpath - // it's okay to convert to u16 here because no codepoint has a width larger - // than 2 and graphemes are usually atmost two visible codepoints wide - Grapheme::Other { ref g } => grapheme_width(g), - Grapheme::Tab { width } => width, - Grapheme::Newline => 1, - } - } - - pub fn is_whitespace(&self) -> bool { - !matches!(&self, Grapheme::Other { g } if !g.chars().next().is_some_and(char_is_whitespace)) - } - - // TODO currently word boundaries are used for softwrapping. - // This works best for programming languages and well for prose. - // This could however be improved in the future by considering unicode - // character classes but - pub fn is_word_boundary(&self) -> bool { - !matches!(&self, Grapheme::Other { g,.. } if g.chars().next().is_some_and(char_is_word)) - } -} - -impl Display for Grapheme<'_> { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match *self { - Grapheme::Newline => write!(f, " "), - Grapheme::Tab { width } => { - for _ in 0..width { - write!(f, " ")?; - } - Ok(()) - } - Grapheme::Other { ref g } => { - write!(f, "{g}") - } - } - } -} +use std::fmt; #[must_use] pub fn grapheme_width(g: &str) -> usize { @@ -113,15 +27,10 @@ pub fn grapheme_width(g: &str) -> usize { // We use max(1) here because all grapeheme clusters--even illformed // ones--should have at least some width so they can be edited // properly. - // TODO properly handle unicode width for all codepoints - // example of where unicode width is currently wrong: 🤦🏼♂️ (taken from https://hsivonen.fi/string-length/) UnicodeWidthStr::width(g).max(1) } } -// NOTE: for byte indexing versions of these functions see `RopeSliceExt`'s -// `floor_grapheme_boundary` and `ceil_grapheme_boundary` and the rope grapheme iterators. - #[must_use] pub fn nth_prev_grapheme_boundary(slice: RopeSlice, char_idx: usize, n: usize) -> usize { // Bounds check @@ -211,6 +120,43 @@ pub fn nth_next_grapheme_boundary(slice: RopeSlice, char_idx: usize, n: usize) - chunk_char_idx + tmp } +#[must_use] +pub fn nth_next_grapheme_boundary_byte(slice: RopeSlice, mut byte_idx: usize, n: usize) -> usize { + // Bounds check + debug_assert!(byte_idx <= slice.len_bytes()); + + // Get the chunk with our byte index in it. + let (mut chunk, mut chunk_byte_idx, mut _chunk_char_idx, _) = slice.chunk_at_byte(byte_idx); + + // Set up the grapheme cursor. + let mut gc = GraphemeCursor::new(byte_idx, slice.len_bytes(), true); + + // Find the nth next grapheme cluster boundary. + for _ in 0..n { + loop { + match gc.next_boundary(chunk, chunk_byte_idx) { + Ok(None) => return slice.len_bytes(), + Ok(Some(n)) => { + byte_idx = n; + break; + } + Err(GraphemeIncomplete::NextChunk) => { + chunk_byte_idx += chunk.len(); + let (a, _, _c, _) = slice.chunk_at_byte(chunk_byte_idx); + chunk = a; + // chunk_char_idx = c; + } + Err(GraphemeIncomplete::PreContext(n)) => { + let ctx_chunk = slice.chunk_at_byte(n - 1).0; + gc.provide_context(ctx_chunk, n - ctx_chunk.len()); + } + _ => unreachable!(), + } + } + } + byte_idx +} + /// Finds the next grapheme boundary after the given char position. #[must_use] #[inline(always)] @@ -218,6 +164,13 @@ pub fn next_grapheme_boundary(slice: RopeSlice, char_idx: usize) -> usize { nth_next_grapheme_boundary(slice, char_idx, 1) } +/// Finds the next grapheme boundary after the given byte position. +#[must_use] +#[inline(always)] +pub fn next_grapheme_boundary_byte(slice: RopeSlice, byte_idx: usize) -> usize { + nth_next_grapheme_boundary_byte(slice, byte_idx, 1) +} + /// Returns the passed char index if it's already a grapheme boundary, /// or the next grapheme boundary char index if not. #[must_use] @@ -242,100 +195,149 @@ pub fn ensure_grapheme_boundary_prev(slice: RopeSlice, char_idx: usize) -> usize } } -/// A highly compressed Cow<'a, str> that holds -/// atmost u31::MAX bytes and is readonly -pub struct GraphemeStr<'a> { - ptr: NonNull<u8>, - len: u32, - phantom: PhantomData<&'a str>, +/// Returns the passed byte index if it's already a grapheme boundary, +/// or the next grapheme boundary byte index if not. +#[must_use] +#[inline] +pub fn ensure_grapheme_boundary_next_byte(slice: RopeSlice, byte_idx: usize) -> usize { + if byte_idx == 0 { + byte_idx + } else { + // TODO: optimize so we're not constructing grapheme cursor twice + if is_grapheme_boundary_byte(slice, byte_idx) { + byte_idx + } else { + next_grapheme_boundary_byte(slice, byte_idx) + } + } } -impl GraphemeStr<'_> { - const MASK_OWNED: u32 = 1 << 31; +/// Returns whether the given char position is a grapheme boundary. +#[must_use] +pub fn is_grapheme_boundary(slice: RopeSlice, char_idx: usize) -> bool { + // Bounds check + debug_assert!(char_idx <= slice.len_chars()); - fn compute_len(&self) -> usize { - (self.len & !Self::MASK_OWNED) as usize - } -} + // We work with bytes for this, so convert. + let byte_idx = slice.char_to_byte(char_idx); -impl Deref for GraphemeStr<'_> { - type Target = str; - fn deref(&self) -> &Self::Target { - unsafe { - let bytes = slice::from_raw_parts(self.ptr.as_ptr(), self.compute_len()); - str::from_utf8_unchecked(bytes) - } - } -} + // Get the chunk with our byte index in it. + let (chunk, chunk_byte_idx, _, _) = slice.chunk_at_byte(byte_idx); -impl Drop for GraphemeStr<'_> { - fn drop(&mut self) { - if self.len & Self::MASK_OWNED != 0 { - // free allocation - unsafe { - drop(Box::from_raw(slice::from_raw_parts_mut( - self.ptr.as_ptr(), - self.compute_len(), - ))); + // Set up the grapheme cursor. + let mut gc = GraphemeCursor::new(byte_idx, slice.len_bytes(), true); + + // Determine if the given position is a grapheme cluster boundary. + loop { + match gc.is_boundary(chunk, chunk_byte_idx) { + Ok(n) => return n, + Err(GraphemeIncomplete::PreContext(n)) => { + let (ctx_chunk, ctx_byte_start, _, _) = slice.chunk_at_byte(n - 1); + gc.provide_context(ctx_chunk, ctx_byte_start); } + Err(_) => unreachable!(), } } } -impl<'a> From<&'a str> for GraphemeStr<'a> { - fn from(g: &'a str) -> Self { - GraphemeStr { - ptr: unsafe { NonNull::new_unchecked(g.as_bytes().as_ptr() as *mut u8) }, - len: i32::try_from(g.len()).unwrap() as u32, - phantom: PhantomData, - } - } -} +/// Returns whether the given byte position is a grapheme boundary. +#[must_use] +pub fn is_grapheme_boundary_byte(slice: RopeSlice, byte_idx: usize) -> bool { + // Bounds check + debug_assert!(byte_idx <= slice.len_bytes()); -impl From<String> for GraphemeStr<'_> { - fn from(g: String) -> Self { - let len = g.len(); - let ptr = Box::into_raw(g.into_bytes().into_boxed_slice()) as *mut u8; - GraphemeStr { - ptr: unsafe { NonNull::new_unchecked(ptr) }, - len: (i32::try_from(len).unwrap() as u32) | Self::MASK_OWNED, - phantom: PhantomData, - } - } -} + // Get the chunk with our byte index in it. + let (chunk, chunk_byte_idx, _, _) = slice.chunk_at_byte(byte_idx); + + // Set up the grapheme cursor. + let mut gc = GraphemeCursor::new(byte_idx, slice.len_bytes(), true); -impl<'a> From<Cow<'a, str>> for GraphemeStr<'a> { - fn from(g: Cow<'a, str>) -> Self { - match g { - Cow::Borrowed(g) => g.into(), - Cow::Owned(g) => g.into(), + // Determine if the given position is a grapheme cluster boundary. + loop { + match gc.is_boundary(chunk, chunk_byte_idx) { + Ok(n) => return n, + Err(GraphemeIncomplete::PreContext(n)) => { + let (ctx_chunk, ctx_byte_start, _, _) = slice.chunk_at_byte(n - 1); + gc.provide_context(ctx_chunk, ctx_byte_start); + } + Err(_) => unreachable!(), } } } -impl<T: Deref<Target = str>> PartialEq<T> for GraphemeStr<'_> { - fn eq(&self, other: &T) -> bool { - self.deref() == other.deref() - } +/// An iterator over the graphemes of a `RopeSlice`. +#[derive(Clone)] +pub struct RopeGraphemes<'a> { + text: RopeSlice<'a>, + chunks: Chunks<'a>, + cur_chunk: &'a str, + cur_chunk_start: usize, + cursor: GraphemeCursor, } -impl PartialEq<str> for GraphemeStr<'_> { - fn eq(&self, other: &str) -> bool { - self.deref() == other - } -} -impl Eq for GraphemeStr<'_> {} -impl Debug for GraphemeStr<'_> { + +impl<'a> fmt::Debug for RopeGraphemes<'a> { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - Debug::fmt(self.deref(), f) + f.debug_struct("RopeGraphemes") + .field("text", &self.text) + .field("chunks", &self.chunks) + .field("cur_chunk", &self.cur_chunk) + .field("cur_chunk_start", &self.cur_chunk_start) + // .field("cursor", &self.cursor) + .finish() } } -impl Display for GraphemeStr<'_> { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - Display::fmt(self.deref(), f) + +impl<'a> RopeGraphemes<'a> { + #[must_use] + pub fn new(slice: RopeSlice) -> RopeGraphemes { + let mut chunks = slice.chunks(); + let first_chunk = chunks.next().unwrap_or(""); + RopeGraphemes { + text: slice, + chunks, + cur_chunk: first_chunk, + cur_chunk_start: 0, + cursor: GraphemeCursor::new(0, slice.len_bytes(), true), + } } } -impl Clone for GraphemeStr<'_> { - fn clone(&self) -> Self { - self.deref().to_owned().into() + +impl<'a> Iterator for RopeGraphemes<'a> { + type Item = RopeSlice<'a>; + + fn next(&mut self) -> Option<RopeSlice<'a>> { + let a = self.cursor.cur_cursor(); + let b; + loop { + match self + .cursor + .next_boundary(self.cur_chunk, self.cur_chunk_start) + { + Ok(None) => { + return None; + } + Ok(Some(n)) => { + b = n; + break; + } + Err(GraphemeIncomplete::NextChunk) => { + self.cur_chunk_start += self.cur_chunk.len(); + self.cur_chunk = self.chunks.next().unwrap_or(""); + } + Err(GraphemeIncomplete::PreContext(idx)) => { + let (chunk, byte_idx, _, _) = self.text.chunk_at_byte(idx.saturating_sub(1)); + self.cursor.provide_context(chunk, byte_idx); + } + _ => unreachable!(), + } + } + + if a < self.cur_chunk_start { + Some(self.text.byte_slice(a..b)) + } else { + let a2 = a - self.cur_chunk_start; + let b2 = b - self.cur_chunk_start; + Some((&self.cur_chunk[a2..b2]).into()) + } } } |