Offline, privacy-first grammar checker. Fast, open-source, Rust-powered
feat: condense TL;DR (#1700)
Andrew Dunbar791084301a07
diff --git a/harper-core/src/char_string.rs b/harper-core/src/char_string.rsindex a1e11fae2..8d29384e2 100644--- a/harper-core/src/char_string.rs+++ b/harper-core/src/char_string.rs@@ -25,6 +25,10 @@ pub trait CharStringExt {/// Only normalizes the left side to lowercase and avoids allocations.fn eq_ignore_ascii_case_str(&self, other: &str) -> bool;+ /// Case-insensitive comparison with any of a list of character slices, assuming the right-hand side is lowercase ASCII.+ /// Only normalizes the left side to lowercase and avoids allocations.+ fn eq_any_ignore_ascii_case_chars(&self, others: &[&[char]]) -> bool;+/// Case-insensitive check if the string ends with the given ASCII suffix./// The suffix is assumed to be lowercase.fn ends_with_ignore_ascii_case_chars(&self, suffix: &[char]) -> bool;@@ -83,6 +87,12 @@ impl CharStringExt for [char] {.all(|(a, b)| a.to_ascii_lowercase() == *b)}+ fn eq_any_ignore_ascii_case_chars(&self, others: &[&[char]]) -> bool {+ others+ .iter()+ .any(|chars| self.eq_ignore_ascii_case_chars(chars))+ }+fn ends_with_ignore_ascii_case_str(&self, suffix: &str) -> bool {let suffix_len = suffix.len();if self.len() < suffix_len {diff --git a/harper-core/src/document.rs b/harper-core/src/document.rsindex e7ce7a437..a1746242a 100644--- a/harper-core/src/document.rs+++ b/harper-core/src/document.rs@@ -11,7 +11,7 @@ use crate::patterns::WordSet;use crate::punctuation::Punctuation;use crate::spell::{Dictionary, FstDictionary};use crate::vec_ext::VecExt;-use crate::{FatStringToken, FatToken, Lrc, Token, TokenKind, TokenStringExt};+use crate::{CharStringExt, FatStringToken, FatToken, Lrc, Token, TokenKind, TokenStringExt};use crate::{OrdinalSuffix, Span};/// A document containing some amount of lexed and parsed English text.@@ -138,6 +138,7 @@ impl Document {self.condense_ellipsis();self.condense_latin();self.condense_filename_extensions();+ self.condense_tldr();self.match_quotes();let chunker = burn_chunker();@@ -627,6 +628,59 @@ impl Document {self.tokens.remove_indices(to_remove);}+ /// Condenses "tl;dr" down to a single word token.+ fn condense_tldr(&mut self) {+ if self.tokens.len() < 3 {+ return;+ }++ let mut to_remove = VecDeque::new();+ let mut cursor = 2;++ loop {+ let tl = &self.tokens[cursor - 2];+ let simicolon = &self.tokens[cursor - 1];+ let dr = &self.tokens[cursor];++ let is_tldr_chunk = tl.kind.is_word()+ && tl.span.len() == 2+ && tl+ .span+ .get_content(&self.source)+ .eq_ignore_ascii_case_chars(&['t', 'l'])+ && simicolon.kind.is_semicolon()+ && dr.kind.is_word()+ && dr.span.len() >= 2+ && dr.span.len() <= 3+ && dr+ .span+ .get_content(&self.source)+ .eq_any_ignore_ascii_case_chars(&[&['d', 'r'], &['d', 'r', 's']]);++ if is_tldr_chunk {+ // Update the first token to be the full "tl;dr" as a word+ self.tokens[cursor - 2].span = Span::new(+ self.tokens[cursor - 2].span.start,+ self.tokens[cursor].span.end,+ );++ // Mark the semicolon and "dr" tokens for removal+ to_remove.push_back(cursor - 1);+ to_remove.push_back(cursor);+ }++ // Skip ahead since we've processed these tokens+ cursor += 1;++ if cursor >= self.tokens.len() {+ break;+ }+ }++ // Remove the marked tokens in reverse order to maintain correct indices+ self.tokens.remove_indices(to_remove);+ }+fn uncached_ellipsis_pattern() -> Lrc<Repeating> {let period = SequenceExpr::default().then_period();Lrc::new(Repeating::new(Box::new(period), 2))@@ -993,4 +1047,54 @@ mod tests {assert!(doc.tokens[22].kind.is_unlintable());assert!(doc.tokens[23].kind.is_close_round());}++ #[test]+ fn condense_tldr_uppercase() {+ let doc = Document::new_plain_english_curated("TL;DR");+ assert!(doc.tokens.len() == 1);+ assert!(doc.tokens[0].kind.is_word());+ assert!(doc.tokens[0].span.len() == 5);+ }++ #[test]+ fn condense_tldr_lowercase() {+ let doc = Document::new_plain_english_curated("tl;dr");+ assert!(doc.tokens.len() == 1);+ assert!(doc.tokens[0].kind.is_word());+ }++ #[test]+ fn condense_tldr_mixed_case_1() {+ let doc = Document::new_plain_english_curated("tl;DR");+ assert!(doc.tokens.len() == 1);+ assert!(doc.tokens[0].kind.is_word());+ }++ #[test]+ fn condense_tldr_mixed_case_2() {+ let doc = Document::new_plain_english_curated("TL;Dr");+ assert!(doc.tokens.len() == 1);+ assert!(doc.tokens[0].kind.is_word());+ }++ #[test]+ fn condense_tldr_pural() {+ let doc = Document::new_plain_english_curated(+ "managing the flow between components to produce relevant TL;DRs of current news articles",+ );+ // no token is a punctuation token - only words with whitespace between+ assert!(+ doc.tokens+ .iter()+ .all(|t| t.kind.is_word() || t.kind.is_whitespace())+ );+ // one of the word tokens contains a ';' character+ let tldrs = doc+ .tokens+ .iter()+ .filter(|t| t.span.get_content(&doc.source).contains(&';'))+ .collect_vec();+ assert!(tldrs.len() == 1);+ assert!(tldrs[0].span.get_content_string(&doc.source) == "TL;DRs");+ }}