Offline, privacy-first grammar checker. Fast, open-source, Rust-powered
fix: hunspell dictionary builder now emits added word metadata
Elijah Potterb4f7c342344d
diff --git a/harper-core/affixes.json b/harper-core/affixes.jsonindex 578ac65f5..063e5b8f0 100644--- a/harper-core/affixes.json+++ b/harper-core/affixes.json@@ -12,7 +12,8 @@],"adds_metadata": {"kind": null,- "tense": null+ "tense": null,+ "plural": null}},"L": {@@ -27,7 +28,8 @@],"adds_metadata": {"kind": null,- "tense": null+ "tense": null,+ "plural": null}},"E": {@@ -42,7 +44,8 @@],"adds_metadata": {"kind": null,- "tense": null+ "tense": null,+ "plural": null}},"Y": {@@ -57,7 +60,8 @@],"adds_metadata": {"kind": null,- "tense": null+ "tense": null,+ "plural": null}},"U": {@@ -72,7 +76,8 @@],"adds_metadata": {"kind": null,- "tense": null+ "tense": null,+ "plural": null}},"H": {@@ -92,7 +97,8 @@],"adds_metadata": {"kind": null,- "tense": null+ "tense": null,+ "plural": null}},"T": {@@ -122,7 +128,8 @@],"adds_metadata": {"kind": null,- "tense": null+ "tense": null,+ "plural": null}},"R": {@@ -152,7 +159,8 @@],"adds_metadata": {"kind": null,- "tense": null+ "tense": null,+ "plural": null}},"C": {@@ -167,7 +175,8 @@],"adds_metadata": {"kind": null,- "tense": null+ "tense": null,+ "plural": null}},"V": {@@ -187,7 +196,8 @@],"adds_metadata": {"kind": null,- "tense": null+ "tense": null,+ "plural": null}},"N": {@@ -212,7 +222,8 @@],"adds_metadata": {"kind": null,- "tense": null+ "tense": null,+ "plural": null}},"A": {@@ -227,7 +238,8 @@],"adds_metadata": {"kind": null,- "tense": null+ "tense": null,+ "plural": null}},"Z": {@@ -257,7 +269,8 @@],"adds_metadata": {"kind": null,- "tense": null+ "tense": null,+ "plural": null}},"P": {@@ -282,7 +295,8 @@],"adds_metadata": {"kind": null,- "tense": null+ "tense": null,+ "plural": null}},"M": {@@ -297,7 +311,8 @@],"adds_metadata": {"kind": null,- "tense": null+ "tense": null,+ "plural": null}},"F": {@@ -312,7 +327,8 @@],"adds_metadata": {"kind": null,- "tense": null+ "tense": null,+ "plural": null}},"B": {@@ -337,7 +353,8 @@],"adds_metadata": {"kind": null,- "tense": null+ "tense": null,+ "plural": null}},"S": {@@ -367,7 +384,8 @@],"adds_metadata": {"kind": null,- "tense": null+ "tense": null,+ "plural": true}},"D": {@@ -397,7 +415,8 @@],"adds_metadata": {"kind": null,- "tense": null+ "tense": null,+ "plural": null}},"G": {@@ -417,7 +436,8 @@],"adds_metadata": {"kind": null,- "tense": null+ "tense": null,+ "plural": null}},"Q": {@@ -432,7 +452,8 @@],"adds_metadata": {"kind": null,- "tense": null+ "tense": null,+ "plural": null}},"O": {@@ -447,7 +468,8 @@],"adds_metadata": {"kind": null,- "tense": null+ "tense": null,+ "plural": null}},"I": {@@ -462,7 +484,8 @@],"adds_metadata": {"kind": null,- "tense": null+ "tense": null,+ "plural": null}},"X": {@@ -487,7 +510,8 @@],"adds_metadata": {"kind": null,- "tense": null+ "tense": null,+ "plural": null}},"J": {@@ -507,8 +531,9 @@],"adds_metadata": {"kind": null,- "tense": null+ "tense": null,+ "plural": null}}}-}\ No newline at end of file+}diff --git a/harper-core/src/spell/full_dictionary.rs b/harper-core/src/spell/full_dictionary.rsindex c264176ac..beb96597f 100644--- a/harper-core/src/spell/full_dictionary.rs+++ b/harper-core/src/spell/full_dictionary.rs@@ -1,11 +1,11 @@-use hashbrown::HashSet;+use hashbrown::{HashMap, HashSet};use once_cell::sync::Lazy;use smallvec::{SmallVec, ToSmallVec};use super::dictionary::Dictionary;use super::hunspell::{parse_default_attribute_list, parse_default_word_list};use super::seq_to_normalized;-use crate::CharString;+use crate::{CharString, WordMetadata};/// A full, fat dictionary./// All of the elements are stored in-memory.@@ -23,7 +23,7 @@ pub struct FullDictionary {/// that has a word whose index is that length.word_len_starts: Vec<usize>,/// All English words- word_set: HashSet<CharString>,+ word_map: HashMap<CharString, WordMetadata>}fn uncached_inner_new() -> FullDictionary {@@ -31,17 +31,18 @@ fn uncached_inner_new() -> FullDictionary {let attr_list = parse_default_attribute_list();// There will be at _least_ this number of words- let mut words = Vec::with_capacity(word_list.len());+ let mut word_map = HashMap::with_capacity(word_list.len());- attr_list.expand_marked_words(word_list, &mut words);+ attr_list.expand_marked_words(word_list, &mut word_map);+ let mut words: Vec<CharString> = word_map.iter().map(|(v, _)| v.clone()).collect();words.sort();words.dedup();FullDictionary {- word_set: HashSet::from_iter(words.iter().cloned()),+ word_map,word_len_starts: FullDictionary::create_len_starts(&mut words),- words,+ words}}@@ -52,7 +53,7 @@ impl FullDictionary {Self {words: Vec::new(),word_len_starts: Vec::new(),- word_set: HashSet::new(),+ word_map: HashMap::new()}}@@ -65,21 +66,26 @@ impl FullDictionary {/// Appends words to the dictionary./// It is significantly faster to append many words with one call than many/// distinct calls to this function.- pub fn extend_words(&mut self, words: impl IntoIterator<Item = impl AsRef<[char]>>) {- let init_size = self.words.len();- self.words- .extend(words.into_iter().map(|v| v.as_ref().to_smallvec()));- self.word_set- .extend(self.words[init_size..].iter().cloned());+ pub fn extend_words(+ &mut self,+ words: impl IntoIterator<Item = (impl AsRef<[char]>, WordMetadata)>+ ) {+ let pairs: Vec<_> = words+ .into_iter()+ .map(|(v, m)| (v.as_ref().to_smallvec(), m))+ .collect();++ self.words.extend(pairs.iter().map(|(v, _)| v.clone()));self.word_len_starts = Self::create_len_starts(&mut self.words);+ self.word_map.extend(pairs);}/// Append a single word to the dictionary.////// If you are appending many words, consider using [`Self::extend_words`]/// instead.- pub fn append_word(&mut self, word: impl AsRef<[char]>) {- self.extend_words(std::iter::once(word.as_ref()))+ pub fn append_word(&mut self, word: impl AsRef<[char]>, metadata: WordMetadata) {+ self.extend_words(std::iter::once((word.as_ref(), metadata)))}/// Create a lookup table for finding words of a specific length in a word@@ -132,7 +138,7 @@ impl Dictionary for FullDictionary {let normalized = seq_to_normalized(word);let lowercase: SmallVec<_> = normalized.iter().flat_map(|c| c.to_lowercase()).collect();- self.word_set.contains(normalized.as_ref()) || self.word_set.contains(&lowercase)+ self.word_map.contains_key(normalized.as_ref()) || self.word_map.contains_key(&lowercase)}}diff --git a/harper-core/src/spell/hunspell/affix_replacement.rs b/harper-core/src/spell/hunspell/affix_replacement.rsindex 17744a590..dcdab2b96 100644--- a/harper-core/src/spell/hunspell/affix_replacement.rs+++ b/harper-core/src/spell/hunspell/affix_replacement.rs@@ -1,12 +1,13 @@use serde::{Deserialize, Serialize};-use super::{matcher::Matcher, Error};+use super::matcher::Matcher;+use super::Error;#[derive(Debug, Clone)]pub struct AffixReplacement {pub remove: Vec<char>,pub add: Vec<char>,- pub condition: Matcher,+ pub condition: Matcher}impl AffixReplacement {@@ -14,18 +15,18 @@ impl AffixReplacement {HumanReadableAffixReplacement {remove: self.remove.iter().collect(),add: self.add.iter().collect(),- condition: self.condition.to_string(),+ condition: self.condition.to_string()}}}-/// A version of [`AffixReplacement`] that can be serialized to JSON (or whatever) and maintain the-/// nice Regex syntax of the inner [`Matcher`].+/// A version of [`AffixReplacement`] that can be serialized to JSON (or+/// whatever) and maintain the nice Regex syntax of the inner [`Matcher`].#[derive(Debug, Clone, Serialize, Deserialize)]pub struct HumanReadableAffixReplacement {pub remove: String,pub add: String,- pub condition: String,+ pub condition: String}impl HumanReadableAffixReplacement {@@ -33,7 +34,7 @@ impl HumanReadableAffixReplacement {Ok(AffixReplacement {remove: self.remove.chars().collect(),add: self.add.chars().collect(),- condition: Matcher::parse(&self.condition)?,+ condition: Matcher::parse(&self.condition)?})}}diff --git a/harper-core/src/spell/hunspell/attribute_list.rs b/harper-core/src/spell/hunspell/attribute_list.rsindex 0626663c4..953b26c0d 100644--- a/harper-core/src/spell/hunspell/attribute_list.rs+++ b/harper-core/src/spell/hunspell/attribute_list.rs@@ -2,16 +2,16 @@ use hashbrown::HashMap;use serde::{Deserialize, Serialize};use smallvec::ToSmallVec;+use super::affix_replacement::AffixReplacement;use super::expansion::{Expansion, HumanReadableExpansion};use super::word_list::MarkedWord;use super::Error;-use super::{affix_replacement::AffixReplacement, matcher::Matcher};use crate::{CharString, Span, WordMetadata};#[derive(Debug, Clone)]pub struct AttributeList {/// Key = Affix Flag- affixes: HashMap<char, Expansion>,+ affixes: HashMap<char, Expansion>}impl AttributeList {@@ -21,7 +21,7 @@ impl AttributeList {.affixes.iter().map(|(affix, exp)| (*affix, exp.to_human_readable()))- .collect(),+ .collect()}}@@ -30,24 +30,26 @@ impl AttributeList {/// Will append to the given `dest`;////// In the future, I want to make this function cleaner and faster.- pub fn expand_marked_word(&self, word: MarkedWord, dest: &mut Vec<CharString>) {+ pub fn expand_marked_word(+ &self,+ word: MarkedWord,+ dest: &mut HashMap<CharString, WordMetadata>+ ) {dest.reserve(word.attributes.len() + 1);- let start_len = dest.len();-for attr in &word.attributes {let Some(expansion) = self.affixes.get(attr) else {continue;};- let mut new_words = Vec::new();+ let mut new_words = HashMap::new();for replacement in &expansion.replacements {- new_words.extend(Self::apply_replacement(- replacement,- &word.letters,- expansion.suffix,- ))+ if let Some(replaced) =+ Self::apply_replacement(replacement, &word.letters, expansion.suffix)+ {+ new_words.insert(replaced, expansion.adds_metadata);+ }}if expansion.cross_product {@@ -62,31 +64,21 @@ impl AttributeList {}}- let cross_product_words = Vec::new();-- for new_word in new_words {+ for (new_word, _metadata) in new_words {self.expand_marked_word(MarkedWord {letters: new_word,- attributes: opp_attr.clone(),+ attributes: opp_attr.clone()},- dest,+ dest);}-- dest.extend_from_slice(&cross_product_words);} else {- dest.extend_from_slice(&new_words);+ dest.extend(new_words.into_iter());}}- dest.push(word.letters);-- let mut split = dest.split_off(start_len);- split.sort();- split.dedup();-- dest.append(&mut split);+ dest.insert(word.letters, WordMetadata::default());}/// Expand an iterator of marked words into strings.@@ -95,7 +87,7 @@ impl AttributeList {pub fn expand_marked_words(&self,words: impl IntoIterator<Item = MarkedWord>,- dest: &mut Vec<CharString>,+ dest: &mut HashMap<CharString, WordMetadata>) {for word in words {self.expand_marked_word(word, dest);@@ -105,7 +97,7 @@ impl AttributeList {fn apply_replacement(replacement: &AffixReplacement,letters: &[char],- suffix: bool,+ suffix: bool) -> Option<CharString> {if replacement.condition.len() > letters.len() {return None;@@ -160,7 +152,7 @@ impl AttributeList {#[derive(Debug, Clone, Serialize, Deserialize)]pub struct HumanReadableAttributeList {- affixes: HashMap<char, HumanReadableExpansion>,+ affixes: HashMap<char, HumanReadableExpansion>}impl HumanReadableAttributeList {diff --git a/harper-core/src/spell/hunspell/expansion.rs b/harper-core/src/spell/hunspell/expansion.rsindex 21a64fdf6..46a86f7f5 100644--- a/harper-core/src/spell/hunspell/expansion.rs+++ b/harper-core/src/spell/hunspell/expansion.rs@@ -1,19 +1,18 @@use serde::{Deserialize, Serialize};+use super::affix_replacement::{AffixReplacement, HumanReadableAffixReplacement};+use super::Error;use crate::WordMetadata;-use super::{- affix_replacement::{AffixReplacement, HumanReadableAffixReplacement},- Error,-};-#[derive(Debug, Clone)]pub struct Expansion {/// If `!true`, this is a prefixpub suffix: bool,pub cross_product: bool,pub replacements: Vec<AffixReplacement>,- pub adds_metadata: WordMetadata,+ /// When the expansion is applied, the resulting word will have fields+ /// overwritten with the none-null properties from here.+ pub adds_metadata: WordMetadata}impl Expansion {@@ -26,7 +25,7 @@ impl Expansion {.iter().map(AffixReplacement::to_human_readable).collect(),- adds_metadata: self.adds_metadata,+ adds_metadata: self.adds_metadata}}}@@ -36,7 +35,7 @@ pub struct HumanReadableExpansion {pub suffix: bool,pub cross_product: bool,pub replacements: Vec<HumanReadableAffixReplacement>,- pub adds_metadata: WordMetadata,+ pub adds_metadata: WordMetadata}impl HumanReadableExpansion {@@ -51,7 +50,7 @@ impl HumanReadableExpansion {suffix: self.suffix,cross_product: self.cross_product,replacements,- adds_metadata: self.adds_metadata,+ adds_metadata: self.adds_metadata})}}diff --git a/harper-core/src/spell/hunspell/matcher.rs b/harper-core/src/spell/hunspell/matcher.rsindex f0574185f..0776afd5a 100644--- a/harper-core/src/spell/hunspell/matcher.rs+++ b/harper-core/src/spell/hunspell/matcher.rs@@ -8,7 +8,7 @@ use serde::{Deserialize, Serialize};#[derive(Debug, Clone, Serialize, Deserialize)]pub struct Matcher {/// Position-based operators.- operators: Vec<Operator>,+ operators: Vec<Operator>}impl Matcher {@@ -42,7 +42,7 @@ impl Matcher {}}'.' => operators.push(Operator::Any),- _ => operators.push(Operator::Literal(c)),+ _ => operators.push(Operator::Literal(c))}char_idx += 1;@@ -93,7 +93,7 @@ impl Display for Matcher {write!(f, "]")?;}- Operator::Any => write!(f, ".")?,+ Operator::Any => write!(f, ".")?}}@@ -106,7 +106,7 @@ enum Operator {Literal(char),MatchOne(Vec<char>),MatchNone(Vec<char>),- Any,+ Any}impl Operator {@@ -115,7 +115,7 @@ impl Operator {Operator::Literal(b) => a == *b,Operator::MatchOne(b) => b.contains(&a),Operator::MatchNone(b) => !b.contains(&a),- Operator::Any => true,+ Operator::Any => true}}}@@ -123,7 +123,7 @@ impl Operator {#[derive(Debug, Clone, Copy, thiserror::Error)]pub enum Error {#[error("Unmatched bracket at index: {index}")]- UnmatchedBracket { index: usize },+ UnmatchedBracket { index: usize }}#[cfg(test)]diff --git a/harper-core/src/spell/hunspell/mod.rs b/harper-core/src/spell/hunspell/mod.rsindex 5a7d2c559..ed7e4989d 100644--- a/harper-core/src/spell/hunspell/mod.rs+++ b/harper-core/src/spell/hunspell/mod.rs@@ -28,12 +28,13 @@ pub fn parse_default_attribute_list() -> AttributeList {#[cfg(test)]mod tests {+ use hashbrown::HashMap;use serde_json::json;use super::word_list::parse_word_list;use super::{parse_default_attribute_list, parse_default_word_list};use crate::spell::hunspell::attribute_list::HumanReadableAttributeList;- use crate::CharString;+ use crate::{CharString, WordMetadata};pub const TEST_WORD_LIST: &str = "3\nhello\ntry/B\nwork/AB";@@ -82,17 +83,17 @@ mod tests {.unwrap();let attributes = attributes.to_normal().unwrap();- let mut expanded = Vec::new();+ let mut expanded = HashMap::new();attributes.expand_marked_words(words, &mut expanded);let expanded: Vec<String> = expanded.into_iter()- .map(|v| v.into_iter().collect())+ .map(|v| v.0.into_iter().collect()).collect();assert_eq!(expanded,- vec!["hello", "tried", "try", "rework", "reworked", "work", "worked",]+ vec!["hello", "tried", "reworked", "rework", "worked", "work", "try"])}@@ -147,18 +148,18 @@ mod tests {.unwrap();let attributes = attributes.to_normal().unwrap();- let mut expanded = Vec::new();+ let mut expanded: HashMap<CharString, WordMetadata> = HashMap::new();attributes.expand_marked_words(words, &mut expanded);- assert!(expanded.contains(&split("giants")))+ assert!(expanded.contains_key(&split("giants")))}- fn build_expanded() -> Vec<CharString> {+ fn build_expanded() -> HashMap<CharString, WordMetadata> {let words = parse_default_word_list().unwrap();let attributes = parse_default_attribute_list();- let mut expanded = Vec::new();+ let mut expanded = HashMap::new();attributes.expand_marked_words(words, &mut expanded);@@ -172,12 +173,12 @@ mod tests {#[test]fn expanded_contains_giants() {- assert!(build_expanded().contains(&split("giants")));+ assert!(build_expanded().contains_key(&split("giants")));}#[test]fn expanded_contains_deallocate() {- assert!(build_expanded().contains(&split("deallocate")));+ assert!(build_expanded().contains_key(&split("deallocate")));}fn split(text: &str) -> CharString {diff --git a/harper-core/src/word_metadata.rs b/harper-core/src/word_metadata.rsindex 2f45af7c7..40c1c8e08 100644--- a/harper-core/src/word_metadata.rs+++ b/harper-core/src/word_metadata.rs@@ -1,22 +1,23 @@use serde::{Deserialize, Serialize};-#[derive(Debug, Default, Copy, Clone, PartialEq, Eq, Serialize, Deserialize)]+#[derive(Debug, Default, Copy, Clone, PartialEq, Eq, Serialize, Deserialize, PartialOrd)]pub struct WordMetadata {kind: Option<WordKind>,tense: Option<Tense>,+ plural: Option<bool>}-#[derive(Debug, Copy, Clone, PartialEq, Eq, Serialize, Deserialize)]+#[derive(Debug, Copy, Clone, PartialEq, Eq, Serialize, Deserialize, PartialOrd)]pub enum Tense {Past,Present,- Future,+ Future}-#[derive(Debug, Copy, Clone, PartialEq, Eq, Serialize, Deserialize)]+#[derive(Debug, Copy, Clone, PartialEq, Eq, Serialize, Deserialize, PartialOrd)]pub enum WordKind {Verb,Noun { is_proper: bool },Adjective,- Adverb,+ Adverb}diff --git a/harper-ls/src/backend.rs b/harper-ls/src/backend.rsindex c613ff7cb..c678d8410 100644--- a/harper-ls/src/backend.rs+++ b/harper-ls/src/backend.rs@@ -13,7 +13,8 @@ use harper_core::{Linter,MergedDictionary,Token,- TokenKind+ TokenKind,+ WordMetadata};use harper_html::HtmlParser;use serde_json::Value;@@ -494,7 +495,7 @@ impl LanguageServer for Backend {let file_url = second.parse().unwrap();let mut dict = self.load_user_dictionary().await;- dict.append_word(word);+ dict.append_word(word, WordMetadata::default());if let Err(err) = self.save_user_dictionary(dict).await {error!("Unable to save user dictionary: {}", err);}@@ -515,7 +516,7 @@ impl LanguageServer for Backend {error!("Unable resolve dictionary path: {file_url}");return Ok(None);};- dict.append_word(word);+ dict.append_word(word, WordMetadata::default());if let Err(err) = self.save_file_dictionary(&file_url, dict).await {error!("Unable to save file dictionary: {}", err);diff --git a/harper-ls/src/dictionary_io.rs b/harper-ls/src/dictionary_io.rsindex 4af97a905..ff3a4bfd1 100644--- a/harper-ls/src/dictionary_io.rs+++ b/harper-ls/src/dictionary_io.rs@@ -1,6 +1,6 @@use std::path::Path;-use harper_core::{Dictionary, FullDictionary};+use harper_core::{Dictionary, FullDictionary, WordMetadata};use tokio::fs::{self, File};use tokio::io::{self, AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt, BufReader, BufWriter};@@ -50,7 +50,10 @@ async fn dict_from_word_list(mut r: impl AsyncRead + Unpin) -> io::Result<FullDir.read_to_string(&mut str).await?;let mut dict = FullDictionary::new();- dict.extend_words(str.lines().map(|l| l.chars().collect::<Vec<char>>()));+ dict.extend_words(+ str.lines()+ .map(|l| (l.chars().collect::<Vec<char>>(), WordMetadata::default()))+ );Ok(dict)}diff --git a/harper-tree-sitter/src/lib.rs b/harper-tree-sitter/src/lib.rsindex 36d347eb7..a94809b68 100644--- a/harper-tree-sitter/src/lib.rs+++ b/harper-tree-sitter/src/lib.rs@@ -1,6 +1,6 @@use std::collections::HashSet;-use harper_core::{FullDictionary, Mask, Masker, Span};+use harper_core::{FullDictionary, Mask, Masker, Span, WordMetadata};use tree_sitter::{Language, Node, Tree, TreeCursor};/// A Harper [`Masker`] that wraps a given tree-sitter language and a condition,@@ -47,7 +47,10 @@ impl TreeSitterMasker {idents.insert(span.get_content(source));}- let idents: Vec<_> = idents.into_iter().collect();+ let idents: Vec<_> = idents+ .into_iter()+ .map(|ident| (ident, WordMetadata::default()))+ .collect();let mut dictionary = FullDictionary::new();dictionary.extend_words(idents);