Offline, privacy-first grammar checker. Fast, open-source, Rust-powered
feat: Flag phrasal verbs spelled as compound nouns (#991)
Andrew Dunbara3fb923fa2b0
diff --git a/harper-core/dictionary.dict b/harper-core/dictionary.dictindex 72fe35175..d0846b796 100644--- a/harper-core/dictionary.dict+++ b/harper-core/dictionary.dict@@ -12486,7 +12486,7 @@ anorak/1MSanorectic/51SManorexia/1Manorexic/51MS-another/~8+another/~8-answer/~14BMDGSanswerable/5Uanswered/~54U@@ -13653,7 +13653,7 @@ backstretch/1MSbackstroke/~14MGDSbacktalk/14Mbacktrack/14SDG-backup/~154MS+backup/~15MSbackward/~51PSYbackwardness/1Mbackwash/14M@@ -17564,7 +17564,7 @@ cleanly/5UTPRjcleanness/1UMcleanse/41ZGDRScleanser/1M-cleanup/~14MS+cleanup/~1MSclear/~541JPTGMDRYSclearance/~1SMclearheaded/5@@ -52867,6 +52867,59 @@ write-off/1MSwrite-up/1MSyear-over-year/5+# Phrasal verbs. These help correct the error where people write "to login" instead of "to log in"+# For this reason the most important ones to include are the ones that have a noun counterpart.+# Remember not to add any affixes since only the verb part at the start undergoes inflection.++back up/4+break up/4+carry out/4+check in/4+check out/4+clean up/4+come back/4+cut back/4+cut off/4+cut over/4+fail over/4+fall back/4+fight back/4+fix up/4+hand out/4+hang out/4+hook up/4+knock out/4+lock down/4+log in/4+log on/4+log off/4+log out/4+look up/4+meet up/4+melt down/4+pick up/4+push back/4+roll back/4+roll out/4+roll over/4+run away/4+set up/4+show off/4+shut down/4+shut off/4+sign in/4+sign off/4+sign out/4+sign up/4+slow down/4+splash down/4+start out/4+start up/4+take down/4+turn out/4+work around/4+work out/4+# New entries may be added below this line# If you're not confident in using the affix annotations, add missing words here# And they will be revieweddiff --git a/harper-core/src/document.rs b/harper-core/src/document.rsindex 5edbad29a..66d19fcf3 100644--- a/harper-core/src/document.rs+++ b/harper-core/src/document.rs@@ -602,28 +602,29 @@ macro_rules! create_fns_on_doc {}impl TokenStringExt for Document {- create_fns_on_doc!(word);- create_fns_on_doc!(hostname);- create_fns_on_doc!(word_like);- create_fns_on_doc!(conjunction);- create_fns_on_doc!(space);+ create_fns_on_doc!(adjective);create_fns_on_doc!(apostrophe);- create_fns_on_doc!(pipe);- create_fns_on_doc!(quote);- create_fns_on_doc!(number);create_fns_on_doc!(at);- create_fns_on_doc!(ellipsis);- create_fns_on_doc!(unlintable);- create_fns_on_doc!(sentence_terminator);- create_fns_on_doc!(paragraph_break);create_fns_on_doc!(chunk_terminator);- create_fns_on_doc!(punctuation);+ create_fns_on_doc!(comma);+ create_fns_on_doc!(conjunction);create_fns_on_doc!(currency);+ create_fns_on_doc!(ellipsis);+ create_fns_on_doc!(hostname);create_fns_on_doc!(likely_homograph);- create_fns_on_doc!(comma);- create_fns_on_doc!(adjective);- create_fns_on_doc!(verb);+ create_fns_on_doc!(noun);+ create_fns_on_doc!(number);+ create_fns_on_doc!(paragraph_break);+ create_fns_on_doc!(pipe);create_fns_on_doc!(preposition);+ create_fns_on_doc!(punctuation);+ create_fns_on_doc!(quote);+ create_fns_on_doc!(sentence_terminator);+ create_fns_on_doc!(space);+ create_fns_on_doc!(unlintable);+ create_fns_on_doc!(verb);+ create_fns_on_doc!(word);+ create_fns_on_doc!(word_like);fn first_sentence_word(&self) -> Option<&Token> {self.tokens.first_sentence_word()diff --git a/harper-core/src/linting/lint_group.rs b/harper-core/src/linting/lint_group.rsindex 15d62052e..e4086b8d9 100644--- a/harper-core/src/linting/lint_group.rs+++ b/harper-core/src/linting/lint_group.rs@@ -48,6 +48,7 @@ use super::of_course::OfCourse;use super::out_of_date::OutOfDate;use super::oxymorons::Oxymorons;use super::pattern_linter::run_on_chunk;+use super::phrasal_verb_as_compound_noun::PhrasalVerbAsCompoundNoun;use super::pique_interest::PiqueInterest;use super::possessive_your::PossessiveYour;use super::pronoun_contraction::PronounContraction;@@ -302,62 +303,65 @@ impl LintGroup {// Add all the more complex rules to the group.insert_struct_rule!(AdjectiveOfA, true);+ insert_struct_rule!(AnA, true);+ insert_struct_rule!(AvoidCurses, true);insert_pattern_rule!(BackInTheDay, true);+ insert_pattern_rule!(BoringWords, false);+ insert_struct_rule!(CapitalizePersonalPronouns, true);+ insert_pattern_rule!(ChockFull, true);+ insert_struct_rule!(CommaFixes, true);+ insert_struct_rule!(CompoundNouns, true);+ insert_pattern_rule!(Confident, true);+ insert_struct_rule!(CorrectNumberSuffix, true);+ insert_struct_rule!(CurrencyPlacement, true);insert_pattern_rule!(Dashes, true);- insert_struct_rule!(WordPressDotcom, true);- insert_pattern_rule!(OutOfDate, true);- insert_struct_rule!(TheMy, true);- insert_pattern_rule!(ThenThan, true);- insert_pattern_rule!(PiqueInterest, true);- insert_pattern_rule!(WasAloud, true);+ insert_pattern_rule!(DespiteOf, true);+ insert_pattern_rule!(DotInitialisms, true);+ insert_struct_rule!(EllipsisLength, true);+ insert_pattern_rule!(ExpandTimeShorthands, true);+ insert_struct_rule!(FirstAidKit, true);+ insert_struct_rule!(ForNoun, true);+ insert_pattern_rule!(Hedging, true);+ insert_pattern_rule!(Hereby, true);+ insert_struct_rule!(HopHope, true);+ insert_struct_rule!(HowTo, true);insert_pattern_rule!(HyphenateNumberDay, true);insert_pattern_rule!(LeftRightHand, true);- insert_struct_rule!(HopHope, true);- insert_pattern_rule!(Hereby, true);+ insert_struct_rule!(LetsConfusion, true);insert_pattern_rule!(Likewise, true);- insert_struct_rule!(CompoundNouns, true);- insert_pattern_rule!(Nobody, true);- insert_pattern_rule!(Whereas, true);- insert_pattern_rule!(PossessiveYour, true);- insert_struct_rule!(SpelledNumbers, false);- insert_struct_rule!(AnA, true);- insert_struct_rule!(UnclosedQuotes, true);- insert_struct_rule!(LongSentences, true);- insert_struct_rule!(RepeatedWords, true);- insert_struct_rule!(Spaces, true);- insert_struct_rule!(CorrectNumberSuffix, true);- insert_struct_rule!(NumberSuffixCapitalization, true);- insert_pattern_rule!(MultipleSequentialPronouns, true);insert_struct_rule!(LinkingVerbs, false);- insert_struct_rule!(AvoidCurses, true);- insert_struct_rule!(EllipsisLength, true);- insert_struct_rule!(CommaFixes, true);- insert_pattern_rule!(DotInitialisms, true);- insert_pattern_rule!(BoringWords, false);- insert_pattern_rule!(UseGenitive, false);- insert_pattern_rule!(ThatWhich, true);- insert_struct_rule!(CapitalizePersonalPronouns, true);+ insert_struct_rule!(LongSentences, true);insert_struct_rule!(MergeWords, true);- insert_struct_rule!(OxfordComma, true);+ insert_pattern_rule!(ModalOf, true);+ insert_pattern_rule!(MultipleSequentialPronouns, true);insert_struct_rule!(NoOxfordComma, false);- insert_struct_rule!(PronounContraction, true);- insert_struct_rule!(CurrencyPlacement, true);- insert_pattern_rule!(SomewhatSomething, true);- insert_struct_rule!(LetsConfusion, true);- insert_pattern_rule!(DespiteOf, true);- insert_pattern_rule!(ChockFull, true);- insert_struct_rule!(HowTo, true);+ insert_pattern_rule!(Nobody, true);+ insert_struct_rule!(NumberSuffixCapitalization, true);insert_struct_rule!(OfCourse, true);- insert_struct_rule!(FirstAidKit, true);+ insert_pattern_rule!(OutOfDate, true);+ insert_struct_rule!(OxfordComma, true);+ insert_pattern_rule!(Oxymorons, true);+ insert_struct_rule!(PhrasalVerbAsCompoundNoun, true);+ insert_pattern_rule!(PiqueInterest, true);+ insert_pattern_rule!(PossessiveYour, true);+ insert_struct_rule!(PronounContraction, true);insert_struct_rule!(PronounKnew, true);+ insert_struct_rule!(RepeatedWords, true);+ insert_pattern_rule!(SomewhatSomething, true);+ insert_struct_rule!(Spaces, true);+ insert_struct_rule!(SpelledNumbers, false);+ insert_pattern_rule!(ThatWhich, true);+ insert_pattern_rule!(TheHowWhy, true);insert_struct_rule!(TheHowWhy, true);+ insert_struct_rule!(TheMy, true);+ insert_pattern_rule!(ThenThan, true);+ insert_struct_rule!(UnclosedQuotes, true);+ insert_pattern_rule!(UseGenitive, false);+ insert_pattern_rule!(WasAloud, true);+ insert_pattern_rule!(Whereas, true);+ insert_pattern_rule!(WidelyAccepted, true);insert_struct_rule!(WidelyAccepted, true);- insert_pattern_rule!(Confident, true);- insert_pattern_rule!(Oxymorons, true);- insert_pattern_rule!(Hedging, true);- insert_pattern_rule!(ExpandTimeShorthands, true);- insert_pattern_rule!(ModalOf, true);- insert_pattern_rule!(ForNoun, true);+ insert_struct_rule!(WordPressDotcom, true);out.add("SpellCheck",diff --git a/harper-core/src/linting/mod.rs b/harper-core/src/linting/mod.rsindex 3e956225c..2e60f5d96 100644--- a/harper-core/src/linting/mod.rs+++ b/harper-core/src/linting/mod.rs@@ -51,6 +51,7 @@ mod out_of_date;mod oxford_comma;mod oxymorons;mod pattern_linter;+mod phrasal_verb_as_compound_noun;mod phrase_corrections;mod pique_interest;mod possessive_your;@@ -119,6 +120,7 @@ pub use out_of_date::OutOfDate;pub use oxford_comma::OxfordComma;pub use oxymorons::Oxymorons;pub use pattern_linter::PatternLinter;+pub use phrasal_verb_as_compound_noun::PhrasalVerbAsCompoundNoun;pub use pique_interest::PiqueInterest;pub use possessive_your::PossessiveYour;pub use pronoun_contraction::PronounContraction;diff --git a/harper-core/src/linting/phrasal_verb_as_compound_noun.rs b/harper-core/src/linting/phrasal_verb_as_compound_noun.rsnew file mode 100644index 000000000..f550d29c4--- /dev/null+++ b/harper-core/src/linting/phrasal_verb_as_compound_noun.rs@@ -0,0 +1,375 @@+use std::sync::Arc;++use super::{Lint, LintKind, Linter, Suggestion};+use crate::{Dictionary, Document, FstDictionary, Span, Token, TokenStringExt};++/// Detect phrasal verbs written as compound nouns.+pub struct PhrasalVerbAsCompoundNoun {+ dict: Arc<FstDictionary>,+}++// Helper functions to get tokens before or after the current one+// After is easy since it already handles going past the end+fn get_token_after(doc: &Document, i: usize, n: usize) -> Option<&Token> {+ doc.get_token(i + n)+}++// Before is trickier since we have to handle going past the start+fn get_token_before(doc: &Document, i: usize, n: usize) -> Option<&Token> {+ (i >= n).then(|| doc.get_token(i - n).unwrap())+}++impl PhrasalVerbAsCompoundNoun {+ pub fn new() -> Self {+ Self {+ dict: FstDictionary::curated(),+ }+ }+}++impl Default for PhrasalVerbAsCompoundNoun {+ fn default() -> Self {+ Self {+ dict: FstDictionary::curated(),+ }+ }+}++impl Linter for PhrasalVerbAsCompoundNoun {+ fn lint(&mut self, document: &Document) -> Vec<Lint> {+ let mut lints = Vec::new();+ for i in document.iter_noun_indices() {+ // It would be handy if there could be a dict flag for nouns which are compounds of phrasal verbs.+ // Instead let's use a few heuristics.+ let token = document.get_token(i).unwrap();+ // * Can't also be a proper noun or a real verb.+ if token.kind.is_proper_noun() || token.kind.is_verb() {+ continue;+ }+ let nountok_charsl = document.get_span_content(&token.span);+ // * Can't contain a hyphen or space.+ if nountok_charsl.contains(&'-') || nountok_charsl.contains(&' ') {+ // This means Harper's tokenizer's concept of a word is different to what we expect+ // or has changed since this code was written.+ unreachable!();+ }+ // * Can't contain an apostrophe+ if nountok_charsl.contains(&'\'') || nountok_charsl.contains(&'’') {+ continue;+ }+ // * Must end with the same letters as one of the particles used in phrasal verbs.+ let particle_endings: &[&[char]] = &[+ &['a', 'r', 'o', 'u', 'n', 'd'][..],+ &['b', 'a', 'c', 'k'][..],+ &['d', 'o', 'w', 'n'][..],+ &['i', 'n'][..],+ &['o', 'n'][..],+ &['o', 'f', 'f'][..],+ &['o', 'u', 't'][..],+ &['o', 'v', 'e', 'r'][..],+ &['u', 'p'][..],+ ];++ // * Must not be in the set of known false positives.+ if nountok_charsl == ['g', 'a', 'l', 'l', 'o', 'n'] {+ // eprintln!(+ // "* Rejected '{}' because it's a known false positive",+ // nountok_charsl.iter().collect::<String>()+ // );+ continue;+ }++ let mut found_particle_len = 0;+ if !particle_endings.iter().any(|ending| {+ let ending_len = ending.len();+ if ending_len <= nountok_charsl.len()+ && ending+ .iter()+ .eq(nountok_charsl[nountok_charsl.len() - ending_len..].iter())+ {+ found_particle_len = ending_len;+ true+ } else {+ false+ }+ }) {+ continue;+ }++ let verb_part = &nountok_charsl[..nountok_charsl.len() - found_particle_len];+ let particle_part = &nountok_charsl[nountok_charsl.len() - found_particle_len..];+ let phrasal_verb: String = verb_part+ .iter()+ .chain(std::iter::once(&' '))+ .chain(particle_part.iter())+ .collect();++ // Check if both things are verbs.+ // So far we only have a small number of phrasal verbs in the dictionary.+ let (verb_part_is_verb, phrasal_verb_is_verb) = (+ self.dict+ .get_word_metadata(verb_part)+ .is_some_and(|md| md.verb.is_some()),+ self.dict+ .get_word_metadata_str(&phrasal_verb)+ .is_some_and(|md| md.verb.is_some()),+ );++ // If neither is a verb, then it's not a phrasal verb+ if !verb_part_is_verb && !phrasal_verb_is_verb {+ continue;+ }++ // Now we know it matches the pattern of a phrasal verb erroneously written as a compound noun.+ // But we have to check if it's an actual compound noun rather than an error.+ // For that we need some heuristics based on the surrounding context.+ // Let's try to get the word before and the word after.+ // To do that we have to get the tokens immediately before and after, which we expect to be whitespace.+ let maybe_prev_ws = get_token_before(document, i, 1);+ let maybe_prev_tok = maybe_prev_ws.and_then(|prev_ws| {+ (prev_ws.kind.is_whitespace()).then(|| get_token_before(document, i, 2).unwrap())+ });++ let maybe_next_ws = get_token_after(document, i, 1);+ let maybe_next_tok = maybe_next_ws.and_then(|next_ws| {+ (next_ws.kind.is_whitespace()).then(|| get_token_after(document, i, 2).unwrap())+ });++ // If it's in isolation, a compound noun is fine.+ if maybe_prev_tok.is_none() && maybe_next_tok.is_none() {+ continue;+ }++ let message = match (phrasal_verb_is_verb, verb_part_is_verb) {+ (true, _) => Some("This word should be a phrasal verb, not a compound noun."),+ (false, true) => {+ Some("This word might be a phrasal verb rather than a compound noun.")+ }+ _ => continue,+ };++ if let Some(prev_tok) = maybe_prev_tok {+ if prev_tok.kind.is_adjective() || prev_tok.kind.is_determiner() {+ // eprintln!(+ // "* Rejected '{}' because it follows the {} '{}'",+ // nountok_charsl.iter().collect::<String>(),+ // if prev_tok.kind.is_adjective() {+ // "adjective"+ // } else {+ // "determiner"+ // },+ // document.get_span_content(&prev_tok.span).iter().collect::<String>(),+ // );+ continue;+ }++ // "dictionary lookup" is not a mistake but "couples breakup" is.+ if prev_tok.kind.is_noun() && !prev_tok.kind.is_plural_noun() {+ // eprintln!(+ // "* Rejected '{}' because it follows the singular noun '{}",+ // nountok_charsl.iter().collect::<String>(),+ // document.get_span_content(&prev_tok.span).iter().collect::<String>(),+ // );+ continue;+ }+ }++ lints.push(Lint {+ span: Span::new(token.span.start, token.span.end),+ lint_kind: LintKind::WordChoice,+ suggestions: vec![Suggestion::ReplaceWith(phrasal_verb.chars().collect())],+ message: message.unwrap().to_string(),+ priority: 63,+ });+ }++ lints+ }++ fn description(&self) -> &str {+ "This rule looks for phrasal verbs written as compound nouns."+ }+}++#[cfg(test)]+mod tests {+ use super::PhrasalVerbAsCompoundNoun;+ use crate::linting::tests::{assert_lint_count, assert_suggestion_result};++ #[test]+ fn flag_breakup_and_workout() {+ assert_lint_count(+ "I will never breakup with Gym. We just seem to workout.",+ PhrasalVerbAsCompoundNoun::default(),+ 2,+ );+ }++ #[test]+ fn correct_breakup_and_workout() {+ assert_suggestion_result(+ "I will never breakup with Gym. We just seem to workout.",+ PhrasalVerbAsCompoundNoun::default(),+ "I will never break up with Gym. We just seem to work out.",+ );+ }++ #[test]+ fn dont_flag_random_words_that_happen_to_end_like_a_particle() {+ assert_lint_count("I like bacon.", PhrasalVerbAsCompoundNoun::default(), 0);+ }++ #[test]+ fn dont_flag_non_verb_particles() {+ assert_lint_count("non", PhrasalVerbAsCompoundNoun::default(), 0);+ }++ #[test]+ fn correct_after_i() {+ assert_suggestion_result(+ "I backup",+ PhrasalVerbAsCompoundNoun::default(),+ "I back up",+ );+ }++ #[test]+ fn correct_after_we() {+ assert_suggestion_result(+ "we breakup",+ PhrasalVerbAsCompoundNoun::default(),+ "we break up",+ );+ }++ #[test]+ fn dont_flag_checkin() {+ // It's actually not a noun in English.+ assert_lint_count("checkin", PhrasalVerbAsCompoundNoun::default(), 0);+ }++ #[test]+ fn dont_flag_cleanup() {+ assert_lint_count("cleanup", PhrasalVerbAsCompoundNoun::default(), 0);+ }++ #[test]+ fn correct_after_you_lowercase() {+ assert_suggestion_result(+ "you checkout",+ PhrasalVerbAsCompoundNoun::default(),+ "you check out",+ );+ }++ #[test]+ fn correct_after_you_capitalized() {+ assert_suggestion_result(+ "You checkout",+ PhrasalVerbAsCompoundNoun::default(),+ "You check out",+ );+ }++ #[test]+ fn flag_checkout_after_you() {+ assert_lint_count("you checkout", PhrasalVerbAsCompoundNoun::default(), 1);+ }++ #[test]+ fn correct_after_they_lowercase() {+ assert_suggestion_result(+ "they cleanup",+ PhrasalVerbAsCompoundNoun::default(),+ "they clean up",+ );+ }++ #[test]+ fn flag_cleanup_after_they() {+ assert_lint_count("they cleanup", PhrasalVerbAsCompoundNoun::default(), 1);+ }++ #[test]+ fn dont_flag_dictionary_lookup() {+ assert_lint_count("dictionary lookup", PhrasalVerbAsCompoundNoun::default(), 0);+ }++ #[test]+ fn flag_couples_breakup() {+ assert_lint_count("couples breakup", PhrasalVerbAsCompoundNoun::default(), 1);+ }++ #[test]+ fn dont_flag_gallon() {+ assert_lint_count("gallon", PhrasalVerbAsCompoundNoun::default(), 0);+ }++ // Maybe this works by accident because "given" is also an adjective.+ // It should be because "funding" is a noun, but it's a gerund, which makes it also a verb.+ // Still, "given start up" doesn't make sense so maybe this test if fine.+ #[test]+ fn dont_flag_startup_funding() {+ assert_lint_count(+ "Yarvin has actually given startup funding. They hang out and party together",+ PhrasalVerbAsCompoundNoun::default(),+ 0,+ );+ }++ #[test]+ fn dont_flag_huge_markup() {+ assert_lint_count(+ "Sell it back to Russia at a huge markup.",+ PhrasalVerbAsCompoundNoun::default(),+ 0,+ );+ }++ #[test]+ fn dont_flag_another_layoff() {+ assert_lint_count(+ "And now just announced another layoff",+ PhrasalVerbAsCompoundNoun::default(),+ 0,+ );+ }++ // "Shakedown" is a compound noun -- it's part of a comma-separated list with another noun "threat"+ // But this is not easy to check for so is not implemented yet.+ // #[test]+ // fn dont_flag_a_threat_or_shakedown() {+ // assert_lint_count(+ // "Just a threat or Shakedown.",+ // PhrasalVerbAsCompoundNoun::default(),+ // 0,+ // );+ // }++ #[test]+ fn dont_flag_a_flyover() {+ assert_lint_count(+ "if I'm the Brits I'm doing a flyover",+ PhrasalVerbAsCompoundNoun::default(),+ 0,+ );+ }++ #[test]+ fn dont_flag_mafia_style_shakedown() {+ assert_lint_count(+ "Basically it's kind of a mafia style shakedown of Ukraine",+ PhrasalVerbAsCompoundNoun::default(),+ 0,+ );+ }++ #[test]+ fn dont_flag_my_meetup_repository() {+ assert_lint_count(+ "I might have in my Meetup repository",+ PhrasalVerbAsCompoundNoun::default(),+ 0,+ );+ }+}diff --git a/harper-core/src/token_string_ext.rs b/harper-core/src/token_string_ext.rsindex f65d86fea..2ff2a1073 100644--- a/harper-core/src/token_string_ext.rs+++ b/harper-core/src/token_string_ext.rs@@ -55,28 +55,29 @@ pub trait TokenStringExt {/// end of the last element.fn span(&self) -> Option<Span>;- create_decl_for!(word);- create_decl_for!(word_like);- create_decl_for!(conjunction);- create_decl_for!(space);+ create_decl_for!(adjective);create_decl_for!(apostrophe);- create_decl_for!(pipe);- create_decl_for!(quote);- create_decl_for!(number);create_decl_for!(at);+ create_decl_for!(comma);+ create_decl_for!(conjunction);+ create_decl_for!(chunk_terminator);+ create_decl_for!(currency);create_decl_for!(ellipsis);create_decl_for!(hostname);- create_decl_for!(unlintable);- create_decl_for!(sentence_terminator);+ create_decl_for!(likely_homograph);+ create_decl_for!(number);+ create_decl_for!(noun);create_decl_for!(paragraph_break);- create_decl_for!(chunk_terminator);+ create_decl_for!(pipe);+ create_decl_for!(preposition);create_decl_for!(punctuation);- create_decl_for!(currency);- create_decl_for!(likely_homograph);- create_decl_for!(comma);- create_decl_for!(adjective);+ create_decl_for!(quote);+ create_decl_for!(sentence_terminator);+ create_decl_for!(space);+ create_decl_for!(unlintable);create_decl_for!(verb);- create_decl_for!(preposition);+ create_decl_for!(word);+ create_decl_for!(word_like);fn iter_linking_verb_indices(&self) -> impl Iterator<Item = usize> + '_;fn iter_linking_verbs(&self) -> impl Iterator<Item = &Token> + '_;@@ -101,28 +102,29 @@ pub trait TokenStringExt {}impl TokenStringExt for [Token] {- create_fns_for!(word);- create_fns_for!(word_like);- create_fns_for!(hostname);- create_fns_for!(conjunction);- create_fns_for!(space);+ create_fns_for!(adjective);create_fns_for!(apostrophe);- create_fns_for!(pipe);- create_fns_for!(quote);- create_fns_for!(number);create_fns_for!(at);- create_fns_for!(punctuation);- create_fns_for!(ellipsis);- create_fns_for!(unlintable);- create_fns_for!(sentence_terminator);- create_fns_for!(paragraph_break);create_fns_for!(chunk_terminator);+ create_fns_for!(comma);+ create_fns_for!(conjunction);create_fns_for!(currency);+ create_fns_for!(ellipsis);+ create_fns_for!(hostname);create_fns_for!(likely_homograph);- create_fns_for!(comma);- create_fns_for!(adjective);- create_fns_for!(verb);+ create_fns_for!(noun);+ create_fns_for!(number);+ create_fns_for!(paragraph_break);+ create_fns_for!(pipe);create_fns_for!(preposition);+ create_fns_for!(punctuation);+ create_fns_for!(quote);+ create_fns_for!(sentence_terminator);+ create_fns_for!(space);+ create_fns_for!(unlintable);+ create_fns_for!(verb);+ create_fns_for!(word_like);+ create_fns_for!(word);fn first_non_whitespace(&self) -> Option<&Token> {self.iter().find(|t| !t.kind.is_whitespace())