Offline, privacy-first grammar checker. Fast, open-source, Rust-powered
feat: condense file extensions into single token (#1602)
Andrew Dunbard3839605dde8
diff --git a/harper-core/src/document.rs b/harper-core/src/document.rsindex 48bd4aec1..27fa138b0 100644--- a/harper-core/src/document.rs+++ b/harper-core/src/document.rs@@ -137,6 +137,7 @@ impl Document {self.condense_number_suffixes();self.condense_ellipsis();self.condense_latin();+ self.condense_filename_extensions();self.match_quotes();let token_strings: Vec<_> = self@@ -564,6 +565,67 @@ impl Document {self.tokens.remove_indices(to_remove);}+ /// Condenses likely filename extensions down to single tokens.+ fn condense_filename_extensions(&mut self) {+ if self.tokens.len() < 2 {+ return;+ }++ let mut to_remove = VecDeque::new();++ let mut cursor = 1;++ let mut ext_start = None;++ loop {+ let a = self.get_token_offset(cursor, -2);+ let b = &self.tokens[cursor - 1];+ let c = &self.tokens[cursor];+ let d = self.get_token_offset(cursor, 1);++ let is_ext_chunk = a.is_none_or(|t| t.kind.is_whitespace())+ && b.kind.is_period()+ && c.kind.is_word()+ && c.span.len() <= 3+ && d.is_none_or(|t| t.kind.is_whitespace())+ && if d.is_none_or(|t| t.kind.is_whitespace()) {+ let ext_chars = c.span.get_content(&self.source);+ ext_chars.iter().all(|c| c.is_ascii_lowercase())+ || ext_chars.iter().all(|c| c.is_ascii_uppercase())+ } else {+ false+ };++ if is_ext_chunk {+ if ext_start.is_none() {+ ext_start = Some(cursor - 1);+ self.tokens[cursor - 1].kind = TokenKind::Unlintable;+ } else {+ to_remove.push_back(cursor - 1);+ }++ to_remove.push_back(cursor);+ cursor += 1;+ } else {+ if let Some(start) = ext_start {+ let end = self.tokens[cursor - 2].span.end;+ let start_tok: &mut Token = &mut self.tokens[start];+ start_tok.span.end = end;+ }++ ext_start = None;+ }++ cursor += 1;++ if cursor >= self.tokens.len() {+ break;+ }+ }++ self.tokens.remove_indices(to_remove);+ }+fn uncached_ellipsis_pattern() -> Lrc<Repeating> {let period = SequenceExpr::default().then_period();Lrc::new(Repeating::new(Box::new(period), 2))@@ -872,4 +934,47 @@ mod tests {assert!(tok.is_none());}++ #[test]+ fn condenses_filename_extensions() {+ let doc = Document::new_plain_english_curated(".c and .exe and .js");+ assert!(doc.tokens[0].kind.is_unlintable());+ assert!(doc.tokens[4].kind.is_unlintable());+ assert!(doc.tokens[8].kind.is_unlintable());+ }++ #[test]+ fn condense_filename_extension_ok_at_start_and_end() {+ let doc = Document::new_plain_english_curated(".c and .EXE");+ assert!(doc.tokens.len() == 5);+ assert!(doc.tokens[0].kind.is_unlintable());+ assert!(doc.tokens[4].kind.is_unlintable());+ }++ #[test]+ fn doesnt_condense_filename_extensions_with_mixed_case() {+ let doc = Document::new_plain_english_curated(".c and .Exe");+ assert!(doc.tokens.len() == 6);+ assert!(doc.tokens[0].kind.is_unlintable());+ assert!(doc.tokens[4].kind.is_punctuation());+ assert!(doc.tokens[5].kind.is_word());+ }++ #[test]+ fn doesnt_condense_filename_extensions_with_non_letters() {+ let doc = Document::new_plain_english_curated(".COM and .C0M");+ assert!(doc.tokens.len() == 6);+ assert!(doc.tokens[0].kind.is_unlintable());+ assert!(doc.tokens[4].kind.is_punctuation());+ assert!(doc.tokens[5].kind.is_word());+ }++ #[test]+ fn doesnt_condense_filename_extensions_longer_than_three() {+ let doc = Document::new_plain_english_curated(".dll and .dlls");+ assert!(doc.tokens.len() == 6);+ assert!(doc.tokens[0].kind.is_unlintable());+ assert!(doc.tokens[4].kind.is_punctuation());+ assert!(doc.tokens[5].kind.is_word());+ }}