Skip to content

Offline, privacy-first grammar checker. Fast, open-source, Rust-powered

feat: condense R&D and Q&A to single word tokens (#1758)

Andrew Dunbar452268ae8881
diff --git a/harper-core/src/document.rs b/harper-core/src/document.rs
index a1746242a..0a4694417 100644
--- a/harper-core/src/document.rs
+++ b/harper-core/src/document.rs
@@ -139,6 +139,7 @@ impl Document {
self.condense_latin();
self.condense_filename_extensions();
self.condense_tldr();
+ self.condense_ampersand_pairs();
self.match_quotes();
let chunker = burn_chunker();
@@ -681,6 +682,65 @@ impl Document {
self.tokens.remove_indices(to_remove);
}
+ /// Condenses "R&D" or "Q&A" down to a single word token.
+ fn condense_ampersand_pairs(&mut self) {
+ if self.tokens.len() < 3 {
+ return;
+ }
+
+ let mut to_remove = VecDeque::new();
+ // The number of tokens we look at, minus 1
+ let mut cursor = 2;
+
+ loop {
+ let l1 = &self.tokens[cursor - 2];
+ let and = &self.tokens[cursor - 1];
+ let l2 = &self.tokens[cursor];
+
+ let is_letter_amp_letter_chunk = l1.kind.is_word()
+ && l1.span.len() == 1
+ && and.kind.is_ampersand()
+ && l2.kind.is_word()
+ && l2.span.len() == 1;
+
+ if is_letter_amp_letter_chunk {
+ let (l1, l2) = (
+ l1.span.get_content(&self.source).first(),
+ l2.span.get_content(&self.source).first(),
+ );
+
+ let is_valid_pair = match (l1, l2) {
+ (Some(l1), Some(l2)) => {
+ matches!(
+ (l1.to_ascii_lowercase(), l2.to_ascii_lowercase()),
+ ('r', 'd') | ('q', 'a')
+ )
+ }
+ _ => false,
+ };
+
+ if is_valid_pair {
+ self.tokens[cursor - 2].span = Span::new(
+ self.tokens[cursor - 2].span.start,
+ self.tokens[cursor].span.end,
+ );
+ to_remove.push_back(cursor - 1);
+ to_remove.push_back(cursor);
+ }
+ }
+
+ // Skip ahead since we've processed these tokens
+ cursor += 1;
+
+ if cursor >= self.tokens.len() {
+ break;
+ }
+ }
+
+ // Remove the marked tokens in reverse order to maintain correct indices
+ self.tokens.remove_indices(to_remove);
+ }
+
fn uncached_ellipsis_pattern() -> Lrc<Repeating> {
let period = SequenceExpr::default().then_period();
Lrc::new(Repeating::new(Box::new(period), 2))
@@ -1097,4 +1157,52 @@ mod tests {
assert!(tldrs.len() == 1);
assert!(tldrs[0].span.get_content_string(&doc.source) == "TL;DRs");
}
+
+ #[test]
+ fn condense_r_and_d_caps() {
+ let doc = Document::new_plain_english_curated("R&D");
+ assert!(doc.tokens.len() == 1);
+ assert!(doc.tokens[0].kind.is_word());
+ }
+
+ #[test]
+ fn condense_r_and_d_mixed_case() {
+ let doc = Document::new_plain_english_curated("R&d");
+ assert!(doc.tokens.len() == 1);
+ assert!(doc.tokens[0].kind.is_word());
+ }
+
+ #[test]
+ fn condense_r_and_d_lowercase() {
+ let doc = Document::new_plain_english_curated("r&d");
+ assert!(doc.tokens.len() == 1);
+ assert!(doc.tokens[0].kind.is_word());
+ }
+
+ #[test]
+ fn dont_condense_r_and_d_with_spaces() {
+ let doc = Document::new_plain_english_curated("R & D");
+ assert!(doc.tokens.len() == 5);
+ assert!(doc.tokens[0].kind.is_word());
+ assert!(doc.tokens[1].kind.is_whitespace());
+ assert!(doc.tokens[2].kind.is_ampersand());
+ assert!(doc.tokens[3].kind.is_whitespace());
+ assert!(doc.tokens[4].kind.is_word());
+ }
+
+ #[test]
+ fn condense_q_and_a() {
+ let doc =
+ Document::new_plain_english_curated("A Q&A platform software for teams at any scales.");
+ assert!(doc.tokens.len() >= 3);
+ assert!(doc.tokens[2].kind.is_word());
+ assert!(doc.tokens[2].span.get_content_string(&doc.source) == "Q&A");
+ }
+
+ #[test]
+ fn dont_allow_mixed_r_and_d_with_q_and_a() {
+ let doc = Document::new_plain_english_curated("R&A or Q&D");
+ assert!(doc.tokens.len() == 9);
+ assert!(doc.tokens[1].kind.is_ampersand() || doc.tokens[7].kind.is_ampersand());
+ }
}
diff --git a/harper-core/src/token_kind.rs b/harper-core/src/token_kind.rs
index 031f70652..c163130ff 100644
--- a/harper-core/src/token_kind.rs
+++ b/harper-core/src/token_kind.rs
@@ -207,6 +207,10 @@ impl TokenKind {
matches!(self, TokenKind::Punctuation(Punctuation::Semicolon))
}
+ pub fn is_ampersand(&self) -> bool {
+ matches!(self, TokenKind::Punctuation(Punctuation::Ampersand))
+ }
+
// Miscellaneous is-methods
/// Checks whether a token is word-like--meaning it is more complex than punctuation and can