Skip to content

Offline, privacy-first grammar checker. Fast, open-source, Rust-powered

feat: reworked layout of `WordMetadata`

Elijah Potter38cbe413b4e2
diff --git a/harper-core/src/document.rs b/harper-core/src/document.rs
index 83748503e..f1e88c7a5 100644
--- a/harper-core/src/document.rs
+++ b/harper-core/src/document.rs
@@ -533,6 +533,10 @@ macro_rules! create_fns_on_doc {
self.tokens.[< first_ $thing >]()
}
+ fn [< last_ $thing >](&self) -> Option<Token> {
+ self.tokens.[< last_ $thing >]()
+ }
+
fn [<iter_ $thing _indices>](&self) -> impl Iterator<Item = usize> + '_ {
self.tokens.[< iter_ $thing _indices >]()
}
@@ -564,6 +568,14 @@ impl TokenStringExt for Document {
fn span(&self) -> Option<Span> {
self.tokens.span()
}
+
+ fn iter_linking_verb_indices(&self) -> impl Iterator<Item = usize> + '_ {
+ self.tokens.iter_linking_verb_indices()
+ }
+
+ fn iter_linking_verbs(&self) -> impl Iterator<Item = Token> + '_ {
+ self.tokens.iter_linking_verbs()
+ }
}
impl Display for Document {
diff --git a/harper-core/src/lib.rs b/harper-core/src/lib.rs
index e51e41398..498f73ce7 100644
--- a/harper-core/src/lib.rs
+++ b/harper-core/src/lib.rs
@@ -27,7 +27,7 @@ pub use spell::{Dictionary, FullDictionary, MergedDictionary};
pub use sync::Lrc;
pub use token::{FatToken, Token, TokenKind, TokenStringExt};
pub use vec_ext::VecExt;
-pub use word_metadata::{Tense, WordKind, WordMetadata};
+pub use word_metadata::{Tense, WordMetadata};
/// A utility function that removes overlapping lints in a vector,
/// keeping the more important ones.
diff --git a/harper-core/src/spell/dictionary.rs b/harper-core/src/spell/dictionary.rs
index 3fc31d2b7..32afe9411 100644
--- a/harper-core/src/spell/dictionary.rs
+++ b/harper-core/src/spell/dictionary.rs
@@ -1,6 +1,6 @@
use blanket::blanket;
-use crate::WordMetadata;
+use crate::{Lrc, WordMetadata};
#[cfg(not(feature = "concurrent"))]
#[blanket(derive(Rc))]
diff --git a/harper-core/src/spell/full_dictionary.rs b/harper-core/src/spell/full_dictionary.rs
index 16f681dbb..ff2530b02 100644
--- a/harper-core/src/spell/full_dictionary.rs
+++ b/harper-core/src/spell/full_dictionary.rs
@@ -142,13 +142,11 @@ impl Dictionary for FullDictionary {
fn get_word_metadata(&self, word: &[char]) -> WordMetadata {
let normalized = seq_to_normalized(word);
- let lowercase: SmallVec<_> = normalized.iter().flat_map(|c| c.to_lowercase()).collect();
self.word_map
.get(normalized.as_ref())
- .or(self.word_map.get(&lowercase))
- .copied()
- .unwrap_or_default()
+ .cloned()
+ .unwrap_or(WordMetadata::default())
}
fn contains_word(&self, word: &[char]) -> bool {
diff --git a/harper-core/src/spell/hunspell/attribute_list.rs b/harper-core/src/spell/hunspell/attribute_list.rs
index 68c486087..ac672053f 100644
--- a/harper-core/src/spell/hunspell/attribute_list.rs
+++ b/harper-core/src/spell/hunspell/attribute_list.rs
@@ -15,12 +15,12 @@ pub struct AttributeList {
}
impl AttributeList {
- pub fn to_human_readable(&self) -> HumanReadableAttributeList {
+ pub fn into_human_readable(self) -> HumanReadableAttributeList {
HumanReadableAttributeList {
affixes: self
.affixes
- .iter()
- .map(|(affix, exp)| (*affix, exp.to_human_readable()))
+ .into_iter()
+ .map(|(affix, exp)| (affix, exp.into_human_readable()))
.collect()
}
}
@@ -33,7 +33,7 @@ impl AttributeList {
pub fn expand_marked_word(
&self,
word: MarkedWord,
- metadata: WordMetadata,
+ mut metadata: WordMetadata,
dest: &mut HashMap<CharString, WordMetadata>
) {
dest.reserve(word.attributes.len() + 1);
@@ -44,6 +44,7 @@ impl AttributeList {
continue;
};
+ gifted_metadata.append(&expansion.gifts_metadata);
let mut new_words: HashMap<CharString, WordMetadata> = HashMap::new();
for replacement in &expansion.replacements {
@@ -51,12 +52,10 @@ impl AttributeList {
Self::apply_replacement(replacement, &word.letters, expansion.suffix)
{
if let Some(val) = new_words.get_mut(&replaced) {
- *val = val.or(&expansion.adds_metadata);
+ val.append(&expansion.adds_metadata);
} else {
new_words.insert(replaced, expansion.adds_metadata.or(&metadata));
}
-
- gifted_metadata = gifted_metadata.or(&expansion.gifts_metadata);
}
}
@@ -89,7 +88,7 @@ impl AttributeList {
} else {
for (key, value) in new_words.into_iter() {
if let Some(val) = dest.get_mut(&key) {
- *val = val.or(&value);
+ val.append(&value);
} else {
dest.insert(key, value);
}
@@ -97,8 +96,12 @@ impl AttributeList {
}
}
- if !dest.contains_key(&word.letters) {
- dest.insert(word.letters, metadata.or(&gifted_metadata));
+ metadata.append(&gifted_metadata);
+ if let Some(prev_val) = dest.get(&word.letters) {
+ metadata.append(prev_val);
+ dest.insert(word.letters, metadata);
+ } else {
+ dest.insert(word.letters, metadata);
}
}
@@ -177,11 +180,11 @@ pub struct HumanReadableAttributeList {
}
impl HumanReadableAttributeList {
- pub fn to_normal(&self) -> Result<AttributeList, Error> {
+ pub fn into_normal(self) -> Result<AttributeList, Error> {
let mut affixes = HashMap::with_capacity(self.affixes.len());
- for (affix, expansion) in &self.affixes {
- affixes.insert(*affix, expansion.to_normal()?);
+ for (affix, expansion) in self.affixes.into_iter() {
+ affixes.insert(affix, expansion.into_normal()?);
}
Ok(AttributeList { affixes })
diff --git a/harper-core/src/spell/hunspell/expansion.rs b/harper-core/src/spell/hunspell/expansion.rs
index d34ae47af..4089e09aa 100644
--- a/harper-core/src/spell/hunspell/expansion.rs
+++ b/harper-core/src/spell/hunspell/expansion.rs
@@ -10,16 +10,16 @@ pub struct Expansion {
pub suffix: bool,
pub cross_product: bool,
pub replacements: Vec<AffixReplacement>,
- /// When the expansion is applied, the resulting word will have fields
- /// overwritten with the non-null properties from here.
+ /// When the expansion is applied, the resulting word will have this
+ /// metadata appended to it.
pub adds_metadata: WordMetadata,
- /// When the expansion is applied, the __parent__ word will have the fields
- /// overwritten with the non-null properties from here.
+ /// When the expansion is applied, the __parent__ word will have this
+ /// metadata appended to it.
pub gifts_metadata: WordMetadata
}
impl Expansion {
- pub fn to_human_readable(&self) -> HumanReadableExpansion {
+ pub fn into_human_readable(self) -> HumanReadableExpansion {
HumanReadableExpansion {
suffix: self.suffix,
cross_product: self.cross_product,
@@ -44,7 +44,7 @@ pub struct HumanReadableExpansion {
}
impl HumanReadableExpansion {
- pub fn to_normal(&self) -> Result<Expansion, Error> {
+ pub fn into_normal(self) -> Result<Expansion, Error> {
let mut replacements = Vec::with_capacity(self.replacements.len());
for replacement in &self.replacements {
diff --git a/harper-core/src/spell/hunspell/mod.rs b/harper-core/src/spell/hunspell/mod.rs
index ec2970604..aac029fc4 100644
--- a/harper-core/src/spell/hunspell/mod.rs
+++ b/harper-core/src/spell/hunspell/mod.rs
@@ -22,7 +22,7 @@ pub fn parse_default_attribute_list() -> AttributeList {
.expect("The built-in affix list should always be valid.");
human_readable
- .to_normal()
+ .into_normal()
.expect("All expressions in the built-in attribute list should be valid.")
}
@@ -34,7 +34,7 @@ mod tests {
use super::word_list::parse_word_list;
use super::{parse_default_attribute_list, parse_default_word_list};
use crate::spell::hunspell::attribute_list::HumanReadableAttributeList;
- use crate::{CharString, WordKind, WordMetadata};
+ use crate::{CharString, WordMetadata};
pub const TEST_WORD_LIST: &str = "3\nhello\ntry/B\nwork/AB";
@@ -83,7 +83,7 @@ mod tests {
}
}))
.unwrap();
- let attributes = attributes.to_normal().unwrap();
+ let attributes = attributes.into_normal().unwrap();
let mut expanded = HashMap::new();
@@ -129,15 +129,12 @@ mod tests {
}
],
"adds_metadata": {
- "kind": {
- "kind": "Noun",
+ "noun": {
"is_plural": true
}
},
"gifts_metadata": {
- "kind": {
- "kind": "Noun",
- }
+ "noun": {}
}
},
"M": {
@@ -150,16 +147,13 @@ mod tests {
"condition": "."
}
],
- "adds_metadata": {
- "kind": null,
- "tense": null
- },
+ "adds_metadata": {},
"gifts_metadata": {}
}
}
}))
.unwrap();
- let attributes = attributes.to_normal().unwrap();
+ let attributes = attributes.into_normal().unwrap();
let mut expanded: HashMap<CharString, WordMetadata> = HashMap::new();
@@ -168,29 +162,11 @@ mod tests {
&mut expanded
);
- assert_eq!(
- expanded.get(&split("giant")),
- Some(&WordMetadata {
- kind: Some(WordKind::Noun {
- is_proper: None,
- is_plural: None
- }),
- tense: None,
- possessive: None
- })
- );
+ let giant_data = expanded.get(&split("giant")).unwrap();
+ assert!(giant_data.is_noun());
- assert_eq!(
- expanded.get(&split("giants")),
- Some(&WordMetadata {
- kind: Some(WordKind::Noun {
- is_proper: None,
- is_plural: Some(true)
- }),
- tense: None,
- possessive: None
- })
- )
+ let giants_data = expanded.get(&split("giants")).unwrap();
+ assert!(giants_data.is_plural_noun());
}
fn build_expanded() -> HashMap<CharString, WordMetadata> {
@@ -233,13 +209,10 @@ mod tests {
#[test]
fn expanded_contains_possessive_abandonment() {
- assert_eq!(
- build_expanded()
- .get(&split("abandonment's"))
- .unwrap()
- .possessive,
- Some(true)
- )
+ assert!(build_expanded()
+ .get(&split("abandonment's"))
+ .unwrap()
+ .is_possessive_noun())
}
#[test]
@@ -249,7 +222,7 @@ mod tests {
let has = expanded.get(&split("has"));
assert!(has.is_some());
- assert!(!matches!(has.unwrap().kind, Some(WordKind::Noun { .. })))
+ assert!(!has.unwrap().is_noun(),)
}
fn split(text: &str) -> CharString {
diff --git a/harper-core/src/spell/merged_dictionary.rs b/harper-core/src/spell/merged_dictionary.rs
index 5e6bbabc9..e137419f4 100644
--- a/harper-core/src/spell/merged_dictionary.rs
+++ b/harper-core/src/spell/merged_dictionary.rs
@@ -1,7 +1,9 @@
use std::sync::Arc;
+use hashbrown::HashMap;
+
use super::dictionary::Dictionary;
-use crate::WordMetadata;
+use crate::{CharString, Lrc, WordMetadata};
/// A simple wrapper over [`Dictionary`] that allows
/// one to merge multiple dictionaries without copying.
@@ -10,7 +12,8 @@ pub struct MergedDictionary<T>
where
T: Dictionary + Clone
{
- children: Vec<Arc<T>>
+ children: Vec<Arc<T>>,
+ merged: HashMap<CharString, WordMetadata>
}
impl<T> MergedDictionary<T>
@@ -19,7 +22,8 @@ where
{
pub fn new() -> Self {
Self {
- children: Vec::new()
+ children: Vec::new(),
+ merged: HashMap::new()
}
}
@@ -53,8 +57,9 @@ where
fn get_word_metadata(&self, word: &[char]) -> WordMetadata {
let mut found_metadata = WordMetadata::default();
for child in &self.children {
- found_metadata = found_metadata.or(&child.get_word_metadata(word));
+ found_metadata.append(&child.get_word_metadata(word));
}
+
found_metadata
}
diff --git a/harper-core/src/token.rs b/harper-core/src/token.rs
index be760b0ac..d521ca1a6 100644
--- a/harper-core/src/token.rs
+++ b/harper-core/src/token.rs
@@ -208,6 +208,8 @@ macro_rules! create_decl_for {
paste! {
fn [< first_ $thing >](&self) -> Option<Token> ;
+ fn [< last_ $thing >](&self) -> Option<Token> ;
+
fn [<iter_ $thing _indices>](&self) -> impl Iterator<Item = usize> + '_;
fn [<iter_ $thing s>](&self) -> impl Iterator<Item = Token> + '_;
@@ -222,6 +224,10 @@ macro_rules! create_fns_for {
self.iter().find(|v| v.kind.[<is_ $thing>]()).copied()
}
+ fn [< last_ $thing >](&self) -> Option<Token> {
+ self.iter().rev().find(|v| v.kind.[<is_ $thing>]()).copied()
+ }
+
fn [<iter_ $thing _indices>](&self) -> impl Iterator<Item = usize> + '_ {
self.iter()
.enumerate()
@@ -250,6 +256,9 @@ pub trait TokenStringExt {
create_decl_for!(quote);
create_decl_for!(number);
create_decl_for!(at);
+
+ fn iter_linking_verb_indices(&self) -> impl Iterator<Item = usize> + '_;
+ fn iter_linking_verbs(&self) -> impl Iterator<Item = Token> + '_;
}
impl TokenStringExt for [Token] {
@@ -282,4 +291,19 @@ impl TokenStringExt for [Token] {
fn span(&self) -> Option<Span> {
Some(Span::new(self.first()?.span.start, self.last()?.span.end))
}
+
+ fn iter_linking_verb_indices(&self) -> impl Iterator<Item = usize> + '_ {
+ self.iter_word_indices().filter(|idx| {
+ let word = self[*idx];
+ let TokenKind::Word(word) = word.kind else {
+ panic!("Should be unreachable.");
+ };
+
+ word.is_linking_verb()
+ })
+ }
+
+ fn iter_linking_verbs(&self) -> impl Iterator<Item = Token> + '_ {
+ self.iter_linking_verb_indices().map(|idx| self[idx])
+ }
}
diff --git a/harper-core/src/word_metadata.rs b/harper-core/src/word_metadata.rs
index 5a53b7638..d244eb5b6 100644
--- a/harper-core/src/word_metadata.rs
+++ b/harper-core/src/word_metadata.rs
@@ -1,37 +1,145 @@
+use is_macro::Is;
use serde::{Deserialize, Serialize};
-#[derive(Debug, Default, Copy, Clone, PartialEq, Eq, Serialize, Deserialize, PartialOrd)]
+#[derive(Debug, Default, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, PartialOrd)]
pub struct WordMetadata {
- pub kind: Option<WordKind>,
- pub tense: Option<Tense>,
- pub possessive: Option<bool>
+ noun: Option<NounData>,
+ verb: Option<VerbData>,
+ adjective: Option<AdjectiveData>,
+ adverb: Option<AdverbData>
}
impl WordMetadata {
+ /// Produce a copy of `self` with the known properties of `other` set.
pub fn or(&self, other: &Self) -> Self {
Self {
- kind: self.kind.or(other.kind),
- tense: self.tense.or(other.tense),
- possessive: self.possessive.or(other.possessive)
+ noun: self.noun.or(other.noun),
+ verb: self.verb.or(other.verb),
+ adjective: self.adjective.or(other.adjective),
+ adverb: self.adverb.or(other.adverb)
}
}
+
+ pub fn is_noun(&self) -> bool {
+ self.noun.is_some()
+ }
+
+ pub fn is_verb(&self) -> bool {
+ self.verb.is_some()
+ }
+
+ pub fn is_adjective(&self) -> bool {
+ self.adjective.is_some()
+ }
+
+ pub fn is_adverb(&self) -> bool {
+ self.adverb.is_some()
+ }
+
+ pub fn is_possessive_noun(&self) -> bool {
+ matches!(
+ self.noun,
+ Some(NounData {
+ is_possessive: Some(true),
+ ..
+ })
+ )
+ }
+
+ pub fn is_plural_noun(&self) -> bool {
+ matches!(
+ self.noun,
+ Some(NounData {
+ is_plural: Some(true),
+ ..
+ })
+ )
+ }
+
+ pub fn is_proper_noun(&self) -> bool {
+ matches!(
+ self.noun,
+ Some(NounData {
+ is_proper: Some(true),
+ ..
+ })
+ )
+ }
+
+ pub fn is_linking_verb(&self) -> bool {
+ matches!(
+ self.verb,
+ Some(VerbData {
+ is_linking: Some(true),
+ ..
+ })
+ )
+ }
+
+ /// Same thing as [`Self::or`], except in-place rather than a copy.
+ pub fn append(&mut self, other: &Self) -> &mut Self {
+ *self = self.or(other);
+ self
+ }
}
-#[derive(Debug, Copy, Clone, PartialEq, Eq, Serialize, Deserialize, PartialOrd)]
+#[derive(Debug, Copy, Clone, PartialEq, Eq, Serialize, Deserialize, PartialOrd, Is)]
pub enum Tense {
Past,
Present,
Future
}
-#[derive(Debug, Copy, Clone, PartialEq, Eq, Serialize, Deserialize, PartialOrd)]
-#[serde(tag = "kind")]
-pub enum WordKind {
- Verb,
- Noun {
- is_proper: Option<bool>,
- is_plural: Option<bool>
- },
- Adjective,
- Adverb
+#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, PartialOrd, Eq)]
+pub struct VerbData {
+ pub is_linking: Option<bool>,
+ pub tense: Option<Tense>
+}
+
+impl VerbData {
+ /// Produce a copy of `self` with the known properties of `other` set.
+ pub fn or(&self, other: &Self) -> Self {
+ Self {
+ is_linking: self.is_linking.or(other.is_linking),
+ tense: self.tense.or(other.tense)
+ }
+ }
+}
+
+#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, PartialOrd, Eq)]
+pub struct NounData {
+ pub is_proper: Option<bool>,
+ pub is_plural: Option<bool>,
+ pub is_possessive: Option<bool>
+}
+
+impl NounData {
+ /// Produce a copy of `self` with the known properties of `other` set.
+ pub fn or(&self, other: &Self) -> Self {
+ Self {
+ is_proper: self.is_proper.or(other.is_proper),
+ is_plural: self.is_plural.or(other.is_plural),
+ is_possessive: self.is_possessive.or(other.is_possessive)
+ }
+ }
+}
+
+#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, PartialOrd, Eq)]
+pub struct AdjectiveData {}
+
+impl AdjectiveData {
+ /// Produce a copy of `self` with the known properties of `other` set.
+ pub fn or(&self, _other: &Self) -> Self {
+ Self {}
+ }
+}
+
+#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, PartialOrd, Eq)]
+pub struct AdverbData {}
+
+impl AdverbData {
+ /// Produce a copy of `self` with the known properties of `other` set.
+ pub fn or(&self, _other: &Self) -> Self {
+ Self {}
+ }
}