Skip to content

Offline, privacy-first grammar checker. Fast, open-source, Rust-powered

feat(core): add word derivation to expansion

Elijah Potterfc7270494876
diff --git a/harper-core/dictionary.dict b/harper-core/dictionary.dict
index 41dabcd27..ba192b21c 100644
--- a/harper-core/dictionary.dict
+++ b/harper-core/dictionary.dict
@@ -26850,7 +26850,7 @@ herd/~14MDRZGS
herder/~1M
herdsman/1M
herdsmen/9
-here/~15Mj
+here/~15j
hereabout/S
hereafter/~15SM
hereby/~
@@ -50464,3 +50464,4 @@ cryptographic/5Q
easy peasy/5
un-American/5
BRICS/
+here's/
diff --git a/harper-core/src/spell/fst_dictionary.rs b/harper-core/src/spell/fst_dictionary.rs
index f7ddf8013..0027af964 100644
--- a/harper-core/src/spell/fst_dictionary.rs
+++ b/harper-core/src/spell/fst_dictionary.rs
@@ -240,6 +240,7 @@ mod tests {
use crate::CharStringExt;
use crate::Dictionary;
+ use crate::WordId;
use super::FstDictionary;
@@ -305,4 +306,73 @@ mod tests {
assert!(dict.words.iter().map(|(word, _)| word).all_unique());
}
+
+ #[test]
+ fn contractions_not_derived() {
+ let dict = FstDictionary::curated();
+
+ let contractions = ["there's", "we're", "here's"];
+
+ for contraction in contractions {
+ dbg!(contraction);
+ assert!(
+ dict.get_word_metadata_str(contraction)
+ .unwrap()
+ .derived_from
+ .is_none()
+ )
+ }
+ }
+
+ #[test]
+ fn plural_llamas_derived_from_llama() {
+ let dict = FstDictionary::curated();
+
+ assert_eq!(
+ dict.get_word_metadata_str("llamas")
+ .unwrap()
+ .derived_from
+ .unwrap(),
+ WordId::from_word_str("llama")
+ )
+ }
+
+ #[test]
+ fn plural_cats_derived_from_cat() {
+ let dict = FstDictionary::curated();
+
+ assert_eq!(
+ dict.get_word_metadata_str("cats")
+ .unwrap()
+ .derived_from
+ .unwrap(),
+ WordId::from_word_str("cat")
+ );
+ }
+
+ #[test]
+ fn unhappy_derived_from_happy() {
+ let dict = FstDictionary::curated();
+
+ assert_eq!(
+ dict.get_word_metadata_str("unhappy")
+ .unwrap()
+ .derived_from
+ .unwrap(),
+ WordId::from_word_str("happy")
+ );
+ }
+
+ #[test]
+ fn quickly_derived_from_quick() {
+ let dict = FstDictionary::curated();
+
+ assert_eq!(
+ dict.get_word_metadata_str("quickly")
+ .unwrap()
+ .derived_from
+ .unwrap(),
+ WordId::from_word_str("quick")
+ );
+ }
}
diff --git a/harper-core/src/spell/hunspell/attribute_list.rs b/harper-core/src/spell/hunspell/attribute_list.rs
index ef637cbda..9cf8b2d4b 100644
--- a/harper-core/src/spell/hunspell/attribute_list.rs
+++ b/harper-core/src/spell/hunspell/attribute_list.rs
@@ -7,7 +7,7 @@ use super::Error;
use super::affix_replacement::AffixReplacement;
use super::expansion::{Expansion, HumanReadableExpansion};
use super::word_list::MarkedWord;
-use crate::{CharString, Span, WordMetadata};
+use crate::{CharString, Span, WordId, WordMetadata};
#[derive(Debug, Clone)]
pub struct AttributeList {
@@ -77,9 +77,12 @@ impl AttributeList {
);
let t_metadata = dest.get_metadata_mut_chars(&new_word).unwrap();
t_metadata.append(&metadata);
+ t_metadata.derived_from = Some(WordId::from_word_chars(&word.letters))
}
} else {
- for (key, value) in new_words.into_iter() {
+ for (key, mut value) in new_words.into_iter() {
+ value.derived_from = Some(WordId::from_word_chars(&word.letters));
+
if let Some(val) = dest.get_metadata_mut_chars(&key) {
val.append(&value);
} else {
diff --git a/harper-core/src/spell/mutable_dictionary.rs b/harper-core/src/spell/mutable_dictionary.rs
index 15f46dd7b..cc13a97c2 100644
--- a/harper-core/src/spell/mutable_dictionary.rs
+++ b/harper-core/src/spell/mutable_dictionary.rs
@@ -227,7 +227,6 @@ impl Dictionary for MutableDictionary {
#[cfg(test)]
mod tests {
- use crate::CharString;
use itertools::Itertools;
use crate::{Dictionary, MutableDictionary};
diff --git a/harper-core/src/spell/word_id.rs b/harper-core/src/spell/word_id.rs
index d889f9c3e..9140a3911 100644
--- a/harper-core/src/spell/word_id.rs
+++ b/harper-core/src/spell/word_id.rs
@@ -1,21 +1,22 @@
use std::hash::BuildHasher;
use foldhash::fast::FixedState;
+use serde::{Deserialize, Serialize};
-use crate::CharStringExt;
+use crate::{CharString, CharStringExt};
/// An identifier for a particular word.
///
/// It works by hashing the word it represents, normalized to lowercase.
/// It is meant for situations where you need to refer to a word (or a collection of words),
/// without storing all of accompanying data (like spelling or metadata).
-#[derive(Hash, Copy, Clone, PartialEq, Eq, PartialOrd, Debug)]
+#[derive(Hash, Copy, Clone, PartialEq, Eq, PartialOrd, Debug, Serialize, Deserialize)]
pub struct WordId {
hash: u64,
}
impl WordId {
- /// Create a Word ID from
+ /// Create a Word ID from a character slice.
pub fn from_word_chars(chars: impl AsRef<[char]>) -> Self {
let normalized = chars.as_ref().normalized();
let lower = normalized.to_lower();
@@ -23,4 +24,11 @@ impl WordId {
Self { hash }
}
+
+ /// Create a word ID from a string.
+ /// Requires allocation, so use sparingly.
+ pub fn from_word_str(text: impl AsRef<str>) -> Self {
+ let chars: CharString = text.as_ref().chars().collect();
+ Self::from_word_chars(chars)
+ }
}
diff --git a/harper-core/src/word_metadata.rs b/harper-core/src/word_metadata.rs
index 74bfc06dc..67d7ed4a0 100644
--- a/harper-core/src/word_metadata.rs
+++ b/harper-core/src/word_metadata.rs
@@ -26,7 +26,7 @@ pub struct WordMetadata {
/// Whether the word is considered especially common.
#[serde(default = "default_false")]
pub common: bool,
- #[serde(skip_serializing, skip_deserializing)]
+ #[serde(default = "default_none")]
pub derived_from: Option<WordId>,
}
@@ -35,6 +35,11 @@ fn default_false() -> bool {
false
}
+/// Needed for `serde`
+fn default_none<T>() -> Option<T> {
+ None
+}
+
macro_rules! generate_metadata_queries {
($($category:ident has $($sub:ident),*).*) => {
paste! {