Offline, privacy-first grammar checker. Fast, open-source, Rust-powered
feat(core): add word derivation to expansion
Elijah Potterfc7270494876
diff --git a/harper-core/dictionary.dict b/harper-core/dictionary.dictindex 41dabcd27..ba192b21c 100644--- a/harper-core/dictionary.dict+++ b/harper-core/dictionary.dict@@ -26850,7 +26850,7 @@ herd/~14MDRZGSherder/~1Mherdsman/1Mherdsmen/9-here/~15Mj+here/~15jhereabout/Shereafter/~15SMhereby/~@@ -50464,3 +50464,4 @@ cryptographic/5Qeasy peasy/5un-American/5BRICS/+here's/diff --git a/harper-core/src/spell/fst_dictionary.rs b/harper-core/src/spell/fst_dictionary.rsindex f7ddf8013..0027af964 100644--- a/harper-core/src/spell/fst_dictionary.rs+++ b/harper-core/src/spell/fst_dictionary.rs@@ -240,6 +240,7 @@ mod tests {use crate::CharStringExt;use crate::Dictionary;+ use crate::WordId;use super::FstDictionary;@@ -305,4 +306,73 @@ mod tests {assert!(dict.words.iter().map(|(word, _)| word).all_unique());}++ #[test]+ fn contractions_not_derived() {+ let dict = FstDictionary::curated();++ let contractions = ["there's", "we're", "here's"];++ for contraction in contractions {+ dbg!(contraction);+ assert!(+ dict.get_word_metadata_str(contraction)+ .unwrap()+ .derived_from+ .is_none()+ )+ }+ }++ #[test]+ fn plural_llamas_derived_from_llama() {+ let dict = FstDictionary::curated();++ assert_eq!(+ dict.get_word_metadata_str("llamas")+ .unwrap()+ .derived_from+ .unwrap(),+ WordId::from_word_str("llama")+ )+ }++ #[test]+ fn plural_cats_derived_from_cat() {+ let dict = FstDictionary::curated();++ assert_eq!(+ dict.get_word_metadata_str("cats")+ .unwrap()+ .derived_from+ .unwrap(),+ WordId::from_word_str("cat")+ );+ }++ #[test]+ fn unhappy_derived_from_happy() {+ let dict = FstDictionary::curated();++ assert_eq!(+ dict.get_word_metadata_str("unhappy")+ .unwrap()+ .derived_from+ .unwrap(),+ WordId::from_word_str("happy")+ );+ }++ #[test]+ fn quickly_derived_from_quick() {+ let dict = FstDictionary::curated();++ assert_eq!(+ dict.get_word_metadata_str("quickly")+ .unwrap()+ .derived_from+ .unwrap(),+ WordId::from_word_str("quick")+ );+ }}diff --git a/harper-core/src/spell/hunspell/attribute_list.rs b/harper-core/src/spell/hunspell/attribute_list.rsindex ef637cbda..9cf8b2d4b 100644--- a/harper-core/src/spell/hunspell/attribute_list.rs+++ b/harper-core/src/spell/hunspell/attribute_list.rs@@ -7,7 +7,7 @@ use super::Error;use super::affix_replacement::AffixReplacement;use super::expansion::{Expansion, HumanReadableExpansion};use super::word_list::MarkedWord;-use crate::{CharString, Span, WordMetadata};+use crate::{CharString, Span, WordId, WordMetadata};#[derive(Debug, Clone)]pub struct AttributeList {@@ -77,9 +77,12 @@ impl AttributeList {);let t_metadata = dest.get_metadata_mut_chars(&new_word).unwrap();t_metadata.append(&metadata);+ t_metadata.derived_from = Some(WordId::from_word_chars(&word.letters))}} else {- for (key, value) in new_words.into_iter() {+ for (key, mut value) in new_words.into_iter() {+ value.derived_from = Some(WordId::from_word_chars(&word.letters));+if let Some(val) = dest.get_metadata_mut_chars(&key) {val.append(&value);} else {diff --git a/harper-core/src/spell/mutable_dictionary.rs b/harper-core/src/spell/mutable_dictionary.rsindex 15f46dd7b..cc13a97c2 100644--- a/harper-core/src/spell/mutable_dictionary.rs+++ b/harper-core/src/spell/mutable_dictionary.rs@@ -227,7 +227,6 @@ impl Dictionary for MutableDictionary {#[cfg(test)]mod tests {- use crate::CharString;use itertools::Itertools;use crate::{Dictionary, MutableDictionary};diff --git a/harper-core/src/spell/word_id.rs b/harper-core/src/spell/word_id.rsindex d889f9c3e..9140a3911 100644--- a/harper-core/src/spell/word_id.rs+++ b/harper-core/src/spell/word_id.rs@@ -1,21 +1,22 @@use std::hash::BuildHasher;use foldhash::fast::FixedState;+use serde::{Deserialize, Serialize};-use crate::CharStringExt;+use crate::{CharString, CharStringExt};/// An identifier for a particular word.////// It works by hashing the word it represents, normalized to lowercase./// It is meant for situations where you need to refer to a word (or a collection of words),/// without storing all of accompanying data (like spelling or metadata).-#[derive(Hash, Copy, Clone, PartialEq, Eq, PartialOrd, Debug)]+#[derive(Hash, Copy, Clone, PartialEq, Eq, PartialOrd, Debug, Serialize, Deserialize)]pub struct WordId {hash: u64,}impl WordId {- /// Create a Word ID from+ /// Create a Word ID from a character slice.pub fn from_word_chars(chars: impl AsRef<[char]>) -> Self {let normalized = chars.as_ref().normalized();let lower = normalized.to_lower();@@ -23,4 +24,11 @@ impl WordId {Self { hash }}++ /// Create a word ID from a string.+ /// Requires allocation, so use sparingly.+ pub fn from_word_str(text: impl AsRef<str>) -> Self {+ let chars: CharString = text.as_ref().chars().collect();+ Self::from_word_chars(chars)+ }}diff --git a/harper-core/src/word_metadata.rs b/harper-core/src/word_metadata.rsindex 74bfc06dc..67d7ed4a0 100644--- a/harper-core/src/word_metadata.rs+++ b/harper-core/src/word_metadata.rs@@ -26,7 +26,7 @@ pub struct WordMetadata {/// Whether the word is considered especially common.#[serde(default = "default_false")]pub common: bool,- #[serde(skip_serializing, skip_deserializing)]+ #[serde(default = "default_none")]pub derived_from: Option<WordId>,}@@ -35,6 +35,11 @@ fn default_false() -> bool {false}+/// Needed for `serde`+fn default_none<T>() -> Option<T> {+ None+}+macro_rules! generate_metadata_queries {($($category:ident has $($sub:ident),*).*) => {paste! {