Skip to content

Offline, privacy-first grammar checker. Fast, open-source, Rust-powered

feat: condense I/O and other pairs into a single token (#1803)

Andrew Dunbarfa5525247c6d
diff --git a/harper-core/src/document.rs b/harper-core/src/document.rs
index 0a4694417..875e99893 100644
--- a/harper-core/src/document.rs
+++ b/harper-core/src/document.rs
@@ -140,6 +140,7 @@ impl Document {
self.condense_filename_extensions();
self.condense_tldr();
self.condense_ampersand_pairs();
+ self.condense_slash_pairs();
self.match_quotes();
let chunker = burn_chunker();
@@ -682,28 +683,36 @@ impl Document {
self.tokens.remove_indices(to_remove);
}
- /// Condenses "R&D" or "Q&A" down to a single word token.
- fn condense_ampersand_pairs(&mut self) {
+ /// Allows condensing of delimited pairs of tokens into a single token.
+ ///
+ /// # Arguments
+ ///
+ /// * `is_delimiter` - A function that returns `true` if the token is a delimiter.
+ /// * `valid_pairs` - A slice of tuples representing the valid pairs of tokens to condense.
+ ///
+ fn condense_delimited_pairs<F>(&mut self, is_delimiter: F, valid_pairs: &[(char, char)])
+ where
+ F: Fn(&TokenKind) -> bool,
+ {
if self.tokens.len() < 3 {
return;
}
let mut to_remove = VecDeque::new();
- // The number of tokens we look at, minus 1
let mut cursor = 2;
loop {
let l1 = &self.tokens[cursor - 2];
- let and = &self.tokens[cursor - 1];
+ let delim = &self.tokens[cursor - 1];
let l2 = &self.tokens[cursor];
- let is_letter_amp_letter_chunk = l1.kind.is_word()
+ let is_delimited_chunk = l1.kind.is_word()
&& l1.span.len() == 1
- && and.kind.is_ampersand()
+ && is_delimiter(&delim.kind)
&& l2.kind.is_word()
&& l2.span.len() == 1;
- if is_letter_amp_letter_chunk {
+ if is_delimited_chunk {
let (l1, l2) = (
l1.span.get_content(&self.source).first(),
l2.span.get_content(&self.source).first(),
@@ -711,10 +720,8 @@ impl Document {
let is_valid_pair = match (l1, l2) {
(Some(l1), Some(l2)) => {
- matches!(
- (l1.to_ascii_lowercase(), l2.to_ascii_lowercase()),
- ('r', 'd') | ('q', 'a')
- )
+ let pair = (l1.to_ascii_lowercase(), l2.to_ascii_lowercase());
+ valid_pairs.contains(&pair)
}
_ => false,
};
@@ -729,18 +736,54 @@ impl Document {
}
}
- // Skip ahead since we've processed these tokens
cursor += 1;
-
if cursor >= self.tokens.len() {
break;
}
}
- // Remove the marked tokens in reverse order to maintain correct indices
self.tokens.remove_indices(to_remove);
}
+ // Condenses "ampersand pairs" such as "R&D" or "Q&A" into single tokens.
+ fn condense_ampersand_pairs(&mut self) {
+ self.condense_delimited_pairs(
+ |kind| kind.is_ampersand(),
+ &[
+ ('b', 'b'), // bed & breakfast
+ ('b', 'w'), // black & white
+ ('g', 't'), // gin & tonic
+ ('k', 'r'), // Kernighan & Ritchie
+ ('q', 'a'), // question & answer
+ ('r', 'b'), // rhythm & blues
+ ('r', 'd'), // research & development
+ ('r', 'r'), // rest & relaxation
+ ('s', 'p'), // Standard & Poor's
+ ],
+ );
+ }
+
+ // Condenses "slash pairs" such as "I/O" into single tokens.
+ fn condense_slash_pairs(&mut self) {
+ self.condense_delimited_pairs(
+ |kind| kind.is_slash(),
+ &[
+ ('a', 'c'), // aircon; alternating current
+ ('b', 'w'), // black and white
+ ('c', 'o'), // care of
+ ('d', 'c'), // direct current
+ ('d', 'l'), // download
+ ('i', 'o'), // input/output
+ ('j', 'k'), // just kidding
+ ('n', 'a'), // not applicable
+ ('r', 'c'), // radio control
+ ('s', 'n'), // serial number
+ ('y', 'n'), // yes/no
+ ('y', 'o'), // years old
+ ],
+ );
+ }
+
fn uncached_ellipsis_pattern() -> Lrc<Repeating> {
let period = SequenceExpr::default().then_period();
Lrc::new(Repeating::new(Box::new(period), 2))
@@ -1205,4 +1248,11 @@ mod tests {
assert!(doc.tokens.len() == 9);
assert!(doc.tokens[1].kind.is_ampersand() || doc.tokens[7].kind.is_ampersand());
}
+
+ #[test]
+ fn condense_io() {
+ let doc = Document::new_plain_english_curated("I/O");
+ assert!(doc.tokens.len() == 1);
+ assert!(doc.tokens[0].kind.is_word());
+ }
}
diff --git a/harper-core/src/token_kind.rs b/harper-core/src/token_kind.rs
index de11af5ab..5cfc1bcf9 100644
--- a/harper-core/src/token_kind.rs
+++ b/harper-core/src/token_kind.rs
@@ -215,6 +215,10 @@ impl TokenKind {
matches!(self, TokenKind::Punctuation(Punctuation::Ampersand))
}
+ pub fn is_slash(&self) -> bool {
+ matches!(self, TokenKind::Punctuation(Punctuation::ForwardSlash))
+ }
+
// Miscellaneous is-methods
/// Checks whether a token is word-like--meaning it is more complex than punctuation and can