Offline, privacy-first grammar checker. Fast, open-source, Rust-powered
refactor(core): move word validity into the type
Elijah Potter9678ca4d72d7
diff --git a/harper-core/src/document.rs b/harper-core/src/document.rsindex b83615e7f..5661bdb62 100644--- a/harper-core/src/document.rs+++ b/harper-core/src/document.rs@@ -129,11 +129,8 @@ impl Document {for token in self.tokens.iter_mut() {if let TokenKind::Word(meta) = &mut token.kind {let word_source = token.span.get_content(&self.source);- let found_meta = dictionary- .get_correct_capitalization_of(word_source)- .map(|canonical_caps| dictionary.get_word_metadata(canonical_caps))- .unwrap_or_default();- *meta = meta.or(&found_meta);+ let found_meta = dictionary.get_word_metadata(word_source);+ *meta = found_meta}}}diff --git a/harper-core/src/lexing/mod.rs b/harper-core/src/lexing/mod.rsindex 2b40adf70..d5c6a8475 100644--- a/harper-core/src/lexing/mod.rs+++ b/harper-core/src/lexing/mod.rs@@ -9,7 +9,7 @@ use url::lex_url;use self::email_address::lex_email_address;use crate::char_ext::CharExt;use crate::punctuation::{Punctuation, Quote};-use crate::{Number, TokenKind, WordMetadata};+use crate::{Number, TokenKind};#[derive(Debug)]pub struct FoundToken {@@ -54,7 +54,7 @@ fn lex_word(source: &[char]) -> Option<FoundToken> {} else {Some(FoundToken {next_index: end,- token: TokenKind::Word(WordMetadata::default()),+ token: TokenKind::Word(None),})}}diff --git a/harper-core/src/linting/capitalize_personal_pronouns.rs b/harper-core/src/linting/capitalize_personal_pronouns.rsindex dc50e324c..3a5115cd3 100644--- a/harper-core/src/linting/capitalize_personal_pronouns.rs+++ b/harper-core/src/linting/capitalize_personal_pronouns.rs@@ -11,14 +11,14 @@ impl Linter for CapitalizePersonalPronouns {let mut lints = Vec::new();for tok in document.iter_words() {- if let TokenKind::Word(WordMetadata {+ if let TokenKind::Word(Some(WordMetadata {noun:Some(NounData {is_pronoun: Some(true),..}),..- }) = tok.kind+ })) = tok.kind{if document.get_span_content(tok.span) == ['i'] {lints.push(Lint {diff --git a/harper-core/src/linting/compound_nouns/general_compound_nouns.rs b/harper-core/src/linting/compound_nouns/general_compound_nouns.rsindex 3c720e56e..4ecd1b41f 100644--- a/harper-core/src/linting/compound_nouns/general_compound_nouns.rs+++ b/harper-core/src/linting/compound_nouns/general_compound_nouns.rs@@ -21,7 +21,7 @@ impl Default for GeneralCompoundNouns {fn default() -> Self {let exceptions_pattern = SequencePattern::default().then(Box::new(|tok: &Token, _: &[char]| {- let Some(meta) = tok.kind.as_word() else {+ let Some(Some(meta)) = tok.kind.as_word() else {return false;};@@ -29,7 +29,7 @@ impl Default for GeneralCompoundNouns {})).then_whitespace().then(Box::new(|tok: &Token, _: &[char]| {- let Some(meta) = tok.kind.as_word() else {+ let Some(Some(meta)) = tok.kind.as_word() else {return false;};diff --git a/harper-core/src/linting/linking_verbs.rs b/harper-core/src/linting/linking_verbs.rsindex ad89c4e2f..f5f32ff17 100644--- a/harper-core/src/linting/linking_verbs.rs+++ b/harper-core/src/linting/linking_verbs.rs@@ -17,7 +17,12 @@ impl Linter for LinkingVerbs {let linking_verb_text = document.get_span_content_str(linking_verb.span);if let Some(prev_word) = &chunk[0..idx].last_word() {- if !prev_word.kind.as_word().unwrap().is_noun() {+ if !prev_word+ .kind+ .as_word()+ .unwrap()+ .is_some_and(|m| m.is_noun())+ {output.push(Lint {span: linking_verb.span,lint_kind: LintKind::Miscellaneous,diff --git a/harper-core/src/linting/matcher.rs b/harper-core/src/linting/matcher.rsindex f06a21f39..f69824a64 100644--- a/harper-core/src/linting/matcher.rs+++ b/harper-core/src/linting/matcher.rs@@ -1,5 +1,5 @@use crate::linting::{Lint, LintKind, Linter, Suggestion};-use crate::{CharString, Document, Punctuation, Span, Token, TokenKind, WordMetadata};+use crate::{CharString, Document, Punctuation, Span, Token, TokenKind};#[derive(Debug, PartialEq, PartialOrd, Clone)]struct PatternToken {@@ -35,7 +35,7 @@ macro_rules! vecword {macro_rules! pt {($str:literal) => {PatternToken {- kind: TokenKind::Word(WordMetadata::default()),+ kind: TokenKind::Word(None),content: Some($str.chars().collect()),}};diff --git a/harper-core/src/linting/sentence_capitalization.rs b/harper-core/src/linting/sentence_capitalization.rsindex 7f4414828..4fe9abe15 100644--- a/harper-core/src/linting/sentence_capitalization.rs+++ b/harper-core/src/linting/sentence_capitalization.rs@@ -71,7 +71,7 @@ fn is_full_sentence(toks: &[Token]) -> bool {let mut has_verb = false;for tok in toks {- if let TokenKind::Word(metadata) = tok.kind {+ if let TokenKind::Word(Some(metadata)) = tok.kind {if metadata.is_noun() {has_noun = true;}diff --git a/harper-core/src/patterns/split_compound_word.rs b/harper-core/src/patterns/split_compound_word.rsindex b972a805e..be34c452a 100644--- a/harper-core/src/patterns/split_compound_word.rs+++ b/harper-core/src/patterns/split_compound_word.rs@@ -43,8 +43,7 @@ impl SplitCompoundWord {buffer.extend_from_slice(&a_chars);buffer.extend_from_slice(&b_chars);- if self.dict.contains_word(&buffer) {- let metadata = self.dict.get_word_metadata(&buffer);+ if let Some(metadata) = self.dict.get_word_metadata(&buffer) {if (self.predicate)(metadata) {let correct = self.dict.get_correct_capitalization_of(&buffer).unwrap();buffer.clear();diff --git a/harper-core/src/spell/dictionary.rs b/harper-core/src/spell/dictionary.rsindex 82d1f3ad1..4254893a7 100644--- a/harper-core/src/spell/dictionary.rs+++ b/harper-core/src/spell/dictionary.rs@@ -32,13 +32,11 @@ pub trait Dictionary: Send + Sync {) -> Vec<FuzzyMatchResult>;fn get_correct_capitalization_of(&self, word: &[char]) -> Option<&'_ [char]>;/// Get the associated [`WordMetadata`] for any capitalization of a given word.- /// If the word isn't in the dictionary, the resulting metadata will be- /// empty.- fn get_word_metadata(&self, word: &[char]) -> WordMetadata;+ fn get_word_metadata(&self, word: &[char]) -> Option<WordMetadata>;/// Get the associated [`WordMetadata`] for any capitalization of a given word./// If the word isn't in the dictionary, the resulting metadata will be/// empty.- fn get_word_metadata_str(&self, word: &str) -> WordMetadata;+ fn get_word_metadata_str(&self, word: &str) -> Option<WordMetadata>;/// Iterate over the words in the dictionary.fn words_iter(&self) -> Box<dyn Iterator<Item = &'_ [char]> + Send + '_>;diff --git a/harper-core/src/spell/fst_dictionary.rs b/harper-core/src/spell/fst_dictionary.rsindex 0e24eff61..89bb279c8 100644--- a/harper-core/src/spell/fst_dictionary.rs+++ b/harper-core/src/spell/fst_dictionary.rs@@ -136,11 +136,11 @@ impl Dictionary for FstDictionary {self.full_dict.contains_word_str(word)}- fn get_word_metadata(&self, word: &[char]) -> WordMetadata {+ fn get_word_metadata(&self, word: &[char]) -> Option<WordMetadata> {self.full_dict.get_word_metadata(word)}- fn get_word_metadata_str(&self, word: &str) -> WordMetadata {+ fn get_word_metadata_str(&self, word: &str) -> Option<WordMetadata> {self.full_dict.get_word_metadata_str(word)}diff --git a/harper-core/src/spell/merged_dictionary.rs b/harper-core/src/spell/merged_dictionary.rsindex 97b15b42b..513808668 100644--- a/harper-core/src/spell/merged_dictionary.rs+++ b/harper-core/src/spell/merged_dictionary.rs@@ -64,13 +64,22 @@ impl Dictionary for MergedDictionary {false}- fn get_word_metadata(&self, word: &[char]) -> WordMetadata {+ fn get_word_metadata(&self, word: &[char]) -> Option<WordMetadata> {+ let mut found_anything = false;let mut found_metadata = WordMetadata::default();+for child in &self.children {- found_metadata.append(&child.get_word_metadata(word));+ if let Some(found_item) = child.get_word_metadata(word) {+ found_metadata.append(&found_item);+ found_anything = true;+ }}- found_metadata+ if found_anything {+ Some(found_metadata)+ } else {+ None+ }}fn words_iter(&self) -> Box<dyn Iterator<Item = &'_ [char]> + Send + '_> {@@ -95,7 +104,7 @@ impl Dictionary for MergedDictionary {self.contains_word(&chars)}- fn get_word_metadata_str(&self, word: &str) -> WordMetadata {+ fn get_word_metadata_str(&self, word: &str) -> Option<WordMetadata> {let chars: CharString = word.chars().collect();self.get_word_metadata(&chars)}diff --git a/harper-core/src/spell/mod.rs b/harper-core/src/spell/mod.rsindex 8998cbee2..9f0cf0d0d 100644--- a/harper-core/src/spell/mod.rs+++ b/harper-core/src/spell/mod.rs@@ -165,7 +165,7 @@ mod tests {let common_words = dict.words_iter().filter_map(|word| {- let metadata = dict.get_word_metadata(word);+ let metadata = dict.get_word_metadata(word).unwrap();if metadata.common {Some(FuzzyMatchResult {word,@@ -181,7 +181,7 @@ mod tests {let uncommon_words = dict.words_iter().filter_map(|word| {- let metadata = dict.get_word_metadata(word);+ let metadata = dict.get_word_metadata(word).unwrap();if metadata.common {None} else {@@ -202,7 +202,7 @@ mod tests {let common_first = suggestions.into_iter().take(3)- .all(|word| dict.get_word_metadata(word).common);+ .all(|word| dict.get_word_metadata(word).unwrap().common);assert!(common_first);}diff --git a/harper-core/src/spell/mutable_dictionary.rs b/harper-core/src/spell/mutable_dictionary.rsindex 8c3faef11..3822633d8 100644--- a/harper-core/src/spell/mutable_dictionary.rs+++ b/harper-core/src/spell/mutable_dictionary.rs@@ -160,16 +160,11 @@ impl Default for MutableDictionary {}impl Dictionary for MutableDictionary {- fn get_word_metadata(&self, word: &[char]) -> WordMetadata {+ fn get_word_metadata(&self, word: &[char]) -> Option<WordMetadata> {let normalized = seq_to_normalized(word);- let Some(correct_caps) = self.get_correct_capitalization_of(&normalized) else {- return WordMetadata::default();- };+ let correct_caps = self.get_correct_capitalization_of(&normalized)?;- self.word_map- .get(correct_caps)- .cloned()- .unwrap_or(WordMetadata::default())+ self.word_map.get(correct_caps).cloned()}fn contains_word(&self, word: &[char]) -> bool {@@ -184,7 +179,7 @@ impl Dictionary for MutableDictionary {self.contains_word(&chars)}- fn get_word_metadata_str(&self, word: &str) -> WordMetadata {+ fn get_word_metadata_str(&self, word: &str) -> Option<WordMetadata> {let chars: CharString = word.chars().collect();self.get_word_metadata(&chars)}@@ -252,7 +247,7 @@ impl Dictionary for MutableDictionary {.map(|(word, edit_distance)| FuzzyMatchResult {word,edit_distance,- metadata: self.get_word_metadata(word),+ metadata: self.get_word_metadata(word).unwrap(),}).collect()}@@ -332,22 +327,28 @@ mod tests {#[test]fn this_is_noun() {let dict = MutableDictionary::curated();- assert!(dict.get_word_metadata_str("this").is_noun());- assert!(dict.get_word_metadata_str("This").is_noun());+ assert!(dict.get_word_metadata_str("this").unwrap().is_noun());+ assert!(dict.get_word_metadata_str("This").unwrap().is_noun());}#[test]fn than_is_conjunction() {let dict = MutableDictionary::curated();- assert!(dict.get_word_metadata_str("than").is_conjunction());- assert!(dict.get_word_metadata_str("Than").is_conjunction());+ assert!(dict.get_word_metadata_str("than").unwrap().is_conjunction());+ assert!(dict.get_word_metadata_str("Than").unwrap().is_conjunction());}#[test]fn herself_is_pronoun() {let dict = MutableDictionary::curated();- assert!(dict.get_word_metadata_str("herself").is_pronoun_noun());- assert!(dict.get_word_metadata_str("Herself").is_pronoun_noun());+ assert!(dict+ .get_word_metadata_str("herself")+ .unwrap()+ .is_pronoun_noun());+ assert!(dict+ .get_word_metadata_str("Herself")+ .unwrap()+ .is_pronoun_noun());}#[test]@@ -359,7 +360,7 @@ mod tests {#[test]fn im_is_common() {let dict = MutableDictionary::curated();- assert!(dict.get_word_metadata_str("I'm").common);+ assert!(dict.get_word_metadata_str("I'm").unwrap().common);}#[test]@@ -380,7 +381,10 @@ mod tests {fn there_is_not_a_pronoun() {let dict = MutableDictionary::curated();- assert!(!dict.get_word_metadata_str("there").is_noun());- assert!(!dict.get_word_metadata_str("there").is_pronoun_noun());+ assert!(!dict.get_word_metadata_str("there").unwrap().is_noun());+ assert!(!dict+ .get_word_metadata_str("there")+ .unwrap()+ .is_pronoun_noun());}}diff --git a/harper-core/src/title_case.rs b/harper-core/src/title_case.rsindex 75101ddd0..8c4b2b7da 100644--- a/harper-core/src/title_case.rs+++ b/harper-core/src/title_case.rs@@ -35,7 +35,7 @@ pub fn make_title_case(toks: &[Token], source: &[char], dict: &impl Dictionary)let mut output = toks.span().unwrap().get_content(source).to_vec();while let Some((index, word)) = word_likes.next() {- if let Some(metadata) = word.kind.as_word() {+ if let Some(Some(metadata)) = word.kind.as_word() {if metadata.is_proper_noun() {// Replace it with the dictionary entry verbatim.let orig_text = word.span.get_content(source);@@ -78,7 +78,7 @@ pub fn make_title_case(toks: &[Token], source: &[char], dict: &impl Dictionary)/// Is not responsible for capitalization requirements that are dependent on token position.fn should_capitalize_token(tok: &Token, source: &[char], dict: &impl Dictionary) -> bool {match tok.kind {- TokenKind::Word(mut metadata) => {+ TokenKind::Word(Some(mut metadata)) => {// Only specific conjunctions are not capitalized.lazy_static! {static ref SPECIAL_CONJUNCTIONS: HashSet<Vec<char>> =@@ -91,7 +91,7 @@ fn should_capitalize_token(tok: &Token, source: &[char], dict: &impl Dictionary)let chars = tok.span.get_content(source);let chars_lower = chars.to_lower();- metadata = metadata.or(&dict.get_word_metadata(&chars_lower));+ metadata = metadata.or(&dict.get_word_metadata(&chars_lower).unwrap_or_default());let is_short_preposition = metadata.preposition && tok.span.len() <= 4;diff --git a/harper-core/src/token_kind.rs b/harper-core/src/token_kind.rsindex 707bd7301..bb0fa508e 100644--- a/harper-core/src/token_kind.rs+++ b/harper-core/src/token_kind.rs@@ -8,7 +8,8 @@ use crate::{ConjunctionData, NounData, Number, Punctuation, Quote, WordMetadata})]#[serde(tag = "kind", content = "value")]pub enum TokenKind {- Word(WordMetadata),+ /// `None` if the word does not exist in the dictionary.+ Word(Option<WordMetadata>),Punctuation(Punctuation),Number(Number),/// A sequence of " " spaces.@@ -53,49 +54,49 @@ impl TokenKind {pub fn is_possessive_noun(&self) -> bool {matches!(self,- TokenKind::Word(WordMetadata {+ TokenKind::Word(Some(WordMetadata {noun: Some(NounData {is_possessive: Some(true),..}),..- })+ })))}pub fn is_pronoun(&self) -> bool {matches!(self,- TokenKind::Word(WordMetadata {+ TokenKind::Word(Some(WordMetadata {noun: Some(NounData {is_pronoun: Some(true),..}),..- })+ })))}pub fn is_proper_noun(&self) -> bool {matches!(self,- TokenKind::Word(WordMetadata {+ TokenKind::Word(Some(WordMetadata {noun: Some(NounData {is_proper: Some(true),..}),..- })+ })))}pub fn is_conjunction(&self) -> bool {matches!(self,- TokenKind::Word(WordMetadata {+ TokenKind::Word(Some(WordMetadata {conjunction: Some(ConjunctionData {}),..- })+ })))}@@ -133,7 +134,10 @@ impl TokenKind {}pub fn is_article(&self) -> bool {- matches!(self, TokenKind::Word(WordMetadata { article: true, .. }))+ matches!(+ self,+ TokenKind::Word(Some(WordMetadata { article: true, .. }))+ )}pub fn is_ellipsis(&self) -> bool {@@ -147,30 +151,30 @@ impl TokenKind {pub fn is_adjective(&self) -> bool {matches!(self,- TokenKind::Word(WordMetadata {+ TokenKind::Word(Some(WordMetadata {adjective: Some(_),..- })+ })))}pub fn is_adverb(&self) -> bool {matches!(self,- TokenKind::Word(WordMetadata {+ TokenKind::Word(Some(WordMetadata {adverb: Some(_),..- })+ })))}pub fn is_swear(&self) -> bool {matches!(self,- TokenKind::Word(WordMetadata {+ TokenKind::Word(Some(WordMetadata {swear: Some(true),..- })+ })))}@@ -196,9 +200,9 @@ impl TokenKind {}impl TokenKind {- /// Construct a [`TokenKind::Word`] with no (default) metadata.+ /// Construct a [`TokenKind::Word`] with no metadata.pub fn blank_word() -> Self {- Self::Word(WordMetadata::default())+ Self::Word(None)}}@@ -236,7 +240,7 @@ impl TokenKind {}pub fn is_verb(&self) -> bool {- let TokenKind::Word(metadata) = self else {+ let TokenKind::Word(Some(metadata)) = self else {return false;};@@ -244,7 +248,7 @@ impl TokenKind {}pub fn is_linking_verb(&self) -> bool {- let TokenKind::Word(metadata) = self else {+ let TokenKind::Word(Some(metadata)) = self else {return false;};@@ -252,7 +256,7 @@ impl TokenKind {}pub fn is_not_pronoun_noun(&self) -> bool {- let TokenKind::Word(metadata) = self else {+ let TokenKind::Word(Some(metadata)) = self else {return true;};@@ -260,7 +264,7 @@ impl TokenKind {}pub fn is_not_plural_noun(&self) -> bool {- let TokenKind::Word(metadata) = self else {+ let TokenKind::Word(Some(metadata)) = self else {return true;};@@ -268,7 +272,7 @@ impl TokenKind {}pub fn is_common_word(&self) -> bool {- let TokenKind::Word(metadata) = self else {+ let TokenKind::Word(Some(metadata)) = self else {return true;};@@ -276,7 +280,7 @@ impl TokenKind {}pub fn is_plural_noun(&self) -> bool {- let TokenKind::Word(metadata) = self else {+ let TokenKind::Word(Some(metadata)) = self else {return false;};@@ -284,7 +288,7 @@ impl TokenKind {}pub fn is_noun(&self) -> bool {- let TokenKind::Word(metadata) = self else {+ let TokenKind::Word(Some(metadata)) = self else {return false;};@@ -292,7 +296,7 @@ impl TokenKind {}pub fn is_likely_homograph(&self) -> bool {- let TokenKind::Word(metadata) = self else {+ let TokenKind::Word(Some(metadata)) = self else {return false;};diff --git a/harper-core/src/token_string_ext.rs b/harper-core/src/token_string_ext.rsindex c2ca1edcf..ced049da5 100644--- a/harper-core/src/token_string_ext.rs+++ b/harper-core/src/token_string_ext.rs@@ -1,4 +1,4 @@-use crate::{Span, Token, TokenKind};+use crate::{Span, Token};use itertools::Itertools;use paste::paste;@@ -152,11 +152,11 @@ impl TokenStringExt for [Token] {fn iter_linking_verb_indices(&self) -> impl Iterator<Item = usize> + '_ {self.iter_word_indices().filter(|idx| {let word = self[*idx];- let TokenKind::Word(word) = word.kind else {- panic!("Should be unreachable.");+ let Some(Some(meta)) = word.kind.as_word() else {+ return false;};- word.is_linking_verb()+ meta.is_linking_verb()})}diff --git a/harper-typst/src/lib.rs b/harper-typst/src/lib.rsindex 28288be2d..10262e9d8 100644--- a/harper-typst/src/lib.rs+++ b/harper-typst/src/lib.rs@@ -297,13 +297,13 @@ mod tests {assert!(matches!(token_kinds.as_slice(),&[- TokenKind::Word(WordMetadata {+ TokenKind::Word(Some(WordMetadata {noun: Some(NounData {is_possessive: Some(true),..}),..- }),+ })),TokenKind::Newline(1),TokenKind::Word(_),]diff --git a/harper-typst/src/typst_translator.rs b/harper-typst/src/typst_translator.rsindex 34400b2a4..3fe8e4aec 100644--- a/harper-typst/src/typst_translator.rs+++ b/harper-typst/src/typst_translator.rs@@ -1,7 +1,7 @@use crate::OffsetCursor;use harper_core::{parsers::{PlainEnglish, StrParser},- Punctuation, Token, TokenKind, WordMetadata,+ Punctuation, Token, TokenKind,};use itertools::Itertools;use typst_syntax::{@@ -89,7 +89,7 @@ impl<'a> TypstTranslator<'a> {.filter_map(|item| match item {DestructuringItem::Pattern(pattern) => self.parse_pattern(pattern, offset),DestructuringItem::Named(named) => merge![- token!(named.name(), TokenKind::Word(WordMetadata::default())),+ token!(named.name(), TokenKind::Word(None)),self.parse_pattern(named.pattern(), offset)],DestructuringItem::Spread(spread) => merge![@@ -226,7 +226,7 @@ impl<'a> TypstTranslator<'a> {Expr::Emph(emph) => iter_recurse(&mut emph.body().exprs()),Expr::Link(a) => token!(a, TokenKind::Url),Expr::Ref(a) => {- token!(a, TokenKind::Word(WordMetadata::default()))+ token!(a, TokenKind::Word(None))}Expr::Heading(heading) => iter_recurse(&mut heading.body().exprs()),Expr::List(list_item) => iter_recurse(&mut list_item.body().exprs()),@@ -284,10 +284,7 @@ impl<'a> TypstTranslator<'a> {),Expr::FieldAccess(field_access) => merge![recurse!(field_access.target()),- token!(- field_access.field(),- TokenKind::Word(WordMetadata::default())- )+ token!(field_access.field(), TokenKind::Word(None))],Expr::Let(let_binding) => merge![match let_binding.kind() {