Offline, privacy-first grammar checker. Fast, open-source, Rust-powered
feat: wrote conversion from hunspell dictionary to custom format
Elijah Potter1d9ae5e20883
diff --git a/Cargo.lock b/Cargo.lockindex ee23ec475..ac95a7237 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -538,6 +538,7 @@ dependencies = ["pulldown-cmark","rand","serde",+ "serde_json","smallvec","thiserror","unicode-blocks",@@ -1092,11 +1093,12 @@ dependencies = [[[package]]name = "serde_json"-version = "1.0.117"+version = "1.0.121"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "455182ea6142b14f93f4bc5320a2b31c1f266b66a4a5c858b013302a5d8cbfc3"+checksum = "4ab380d7d9f22ef3f21ad3e6c1ebe8e4fc7a2000ccba2e4d71fc96f15b2cb609"dependencies = ["itoa",+ "memchr","ryu","serde",]diff --git a/harper-core/Cargo.toml b/harper-core/Cargo.tomlindex b2664c0d1..ac9b68212 100644--- a/harper-core/Cargo.toml+++ b/harper-core/Cargo.toml@@ -16,6 +16,7 @@ once_cell = "1.19.0"paste = "1.0.14"pulldown-cmark = "0.11.0"serde = { version = "1.0.197", features = ["derive"] }+serde_json = "1.0.121"smallvec = { version = "1.13.2", features = ["serde"] }thiserror = "1.0.57"unicode-blocks = "0.1.9"diff --git a/harper-core/src/lib.rs b/harper-core/src/lib.rsindex d69e87d8e..fc7be0206 100644--- a/harper-core/src/lib.rs+++ b/harper-core/src/lib.rs@@ -15,6 +15,7 @@ mod spell;mod sync;mod token;mod vec_ext;+mod word_metadata;pub use char_string::{CharString, CharStringExt};pub use document::Document;@@ -26,6 +27,7 @@ pub use spell::{Dictionary, FullDictionary, MergedDictionary};pub use sync::Lrc;pub use token::{FatToken, Token, TokenKind, TokenStringExt};pub use vec_ext::VecExt;+pub use word_metadata::{Tense, WordKind, WordMetadata};/// A utility function that removes overlapping lints in a vector,/// keeping the more important ones.diff --git a/harper-core/src/spell/hunspell/affix_replacement.rs b/harper-core/src/spell/hunspell/affix_replacement.rsnew file mode 100644index 000000000..17744a590--- /dev/null+++ b/harper-core/src/spell/hunspell/affix_replacement.rs@@ -0,0 +1,39 @@+use serde::{Deserialize, Serialize};++use super::{matcher::Matcher, Error};++#[derive(Debug, Clone)]+pub struct AffixReplacement {+ pub remove: Vec<char>,+ pub add: Vec<char>,+ pub condition: Matcher,+}++impl AffixReplacement {+ pub fn to_human_readable(&self) -> HumanReadableAffixReplacement {+ HumanReadableAffixReplacement {+ remove: self.remove.iter().collect(),+ add: self.add.iter().collect(),+ condition: self.condition.to_string(),+ }+ }+}++/// A version of [`AffixReplacement`] that can be serialized to JSON (or whatever) and maintain the+/// nice Regex syntax of the inner [`Matcher`].+#[derive(Debug, Clone, Serialize, Deserialize)]+pub struct HumanReadableAffixReplacement {+ pub remove: String,+ pub add: String,+ pub condition: String,+}++impl HumanReadableAffixReplacement {+ pub fn to_normal(&self) -> Result<AffixReplacement, Error> {+ Ok(AffixReplacement {+ remove: self.remove.chars().collect(),+ add: self.add.chars().collect(),+ condition: Matcher::parse(&self.condition)?,+ })+ }+}diff --git a/harper-core/src/spell/hunspell/attributes.rs b/harper-core/src/spell/hunspell/attribute_list.rssimilarity index 82%rename from harper-core/src/spell/hunspell/attributes.rsrename to harper-core/src/spell/hunspell/attribute_list.rsindex f89886798..7b521b0f9 100644--- a/harper-core/src/spell/hunspell/attributes.rs+++ b/harper-core/src/spell/hunspell/attribute_list.rs@@ -1,36 +1,23 @@use hashbrown::HashMap;+use serde::{Deserialize, Serialize};use smallvec::ToSmallVec;-use super::matcher::Matcher;+use super::expansion::{Expansion, HumanReadableExpansion};use super::word_list::MarkedWord;use super::Error;-use crate::{CharString, Span};--#[derive(Debug, Clone)]-struct AffixReplacement {- pub remove: Vec<char>,- pub add: Vec<char>,- pub condition: Matcher-}--#[derive(Debug, Clone)]-struct Expansion {- // If not true, its a prefix- pub suffix: bool,- pub cross_product: bool,- pub replacements: Vec<AffixReplacement>-}+use super::{affix_replacement::AffixReplacement, matcher::Matcher};+use crate::{CharString, Span, WordMetadata};#[derive(Debug, Clone)]pub struct AttributeList {/// Key = Affix Flag- affixes: HashMap<char, Expansion>+ affixes: HashMap<char, Expansion>,}impl AttributeList {pub fn parse(file: &str) -> Result<Self, Error> {let mut output = Self {- affixes: HashMap::default()+ affixes: HashMap::default(),};for line in file.lines() {@@ -44,6 +31,16 @@ impl AttributeList {Ok(output)}+ pub fn to_human_readable(&self) -> HumanReadableAttributeList {+ HumanReadableAttributeList {+ affixes: self+ .affixes+ .iter()+ .map(|(affix, exp)| (*affix, exp.to_human_readable()))+ .collect(),+ }+ }+fn parse_line(&mut self, line: &str) -> Result<(), Error> {if line.len() < 4 {return Ok(());@@ -54,7 +51,7 @@ impl AttributeList {let suffix = match parser.parse_arg()? {"PFX" => false,"SFX" => true,- _ => return Ok(())+ _ => return Ok(()),};let flag = {@@ -82,7 +79,7 @@ impl AttributeList {let replacement = AffixReplacement {remove,add,- condition+ condition,};expansion.replacements.push(replacement)@@ -95,8 +92,9 @@ impl AttributeList {Expansion {suffix,cross_product,- replacements: Vec::with_capacity(count)- }+ replacements: Vec::with_capacity(count),+ adds_metadata: WordMetadata::default(),+ },);}@@ -124,7 +122,7 @@ impl AttributeList {new_words.extend(Self::apply_replacement(replacement,&word.letters,- expansion.suffix+ expansion.suffix,))}@@ -146,9 +144,9 @@ impl AttributeList {self.expand_marked_word(MarkedWord {letters: new_word,- attributes: opp_attr.clone()+ attributes: opp_attr.clone(),},- dest+ dest,);}@@ -173,7 +171,7 @@ impl AttributeList {pub fn expand_marked_words(&self,words: impl IntoIterator<Item = MarkedWord>,- dest: &mut Vec<CharString>+ dest: &mut Vec<CharString>,) {for word in words {self.expand_marked_word(word, dest);@@ -183,7 +181,7 @@ impl AttributeList {fn apply_replacement(replacement: &AffixReplacement,letters: &[char],- suffix: bool+ suffix: bool,) -> Option<CharString> {if replacement.condition.len() > letters.len() {return None;@@ -236,9 +234,26 @@ impl AttributeList {}}+#[derive(Debug, Clone, Serialize, Deserialize)]+pub struct HumanReadableAttributeList {+ affixes: HashMap<char, HumanReadableExpansion>,+}++impl HumanReadableAttributeList {+ pub fn to_normal(&self) -> Result<AttributeList, Error> {+ let mut affixes = HashMap::with_capacity(self.affixes.len());++ for (affix, expansion) in &self.affixes {+ affixes.insert(*affix, expansion.to_normal()?);+ }++ Ok(AttributeList { affixes })+ }+}+struct AttributeArgParser<'a> {line: &'a str,- cursor: usize+ cursor: usize,}impl<'a> AttributeArgParser<'a> {@@ -246,7 +261,7 @@ impl<'a> AttributeArgParser<'a> {Self { line, cursor: 0 }}- // Grap next affix argument, returning an error if it doesn't exist.+ // Grab next affix argument, returning an error if it doesn't exist.fn parse_arg(&mut self) -> Result<&'a str, Error> {let Some((next_word_start, _)) = self.line[self.cursor..].char_indices()@@ -282,7 +297,7 @@ impl<'a> AttributeArgParser<'a> {match self.parse_arg()? {"Y" => Ok(true),"N" => Ok(false),- _ => Err(Error::ExpectedBoolean)+ _ => Err(Error::ExpectedBoolean),}}}diff --git a/harper-core/src/spell/hunspell/expansion.rs b/harper-core/src/spell/hunspell/expansion.rsnew file mode 100644index 000000000..21a64fdf6--- /dev/null+++ b/harper-core/src/spell/hunspell/expansion.rs@@ -0,0 +1,57 @@+use serde::{Deserialize, Serialize};++use crate::WordMetadata;++use super::{+ affix_replacement::{AffixReplacement, HumanReadableAffixReplacement},+ Error,+};++#[derive(Debug, Clone)]+pub struct Expansion {+ /// If `!true`, this is a prefix+ pub suffix: bool,+ pub cross_product: bool,+ pub replacements: Vec<AffixReplacement>,+ pub adds_metadata: WordMetadata,+}++impl Expansion {+ pub fn to_human_readable(&self) -> HumanReadableExpansion {+ HumanReadableExpansion {+ suffix: self.suffix,+ cross_product: self.cross_product,+ replacements: self+ .replacements+ .iter()+ .map(AffixReplacement::to_human_readable)+ .collect(),+ adds_metadata: self.adds_metadata,+ }+ }+}++#[derive(Debug, Clone, Serialize, Deserialize)]+pub struct HumanReadableExpansion {+ pub suffix: bool,+ pub cross_product: bool,+ pub replacements: Vec<HumanReadableAffixReplacement>,+ pub adds_metadata: WordMetadata,+}++impl HumanReadableExpansion {+ pub fn to_normal(&self) -> Result<Expansion, Error> {+ let mut replacements = Vec::with_capacity(self.replacements.len());++ for replacement in &self.replacements {+ replacements.push(replacement.to_normal()?);+ }++ Ok(Expansion {+ suffix: self.suffix,+ cross_product: self.cross_product,+ replacements,+ adds_metadata: self.adds_metadata,+ })+ }+}diff --git a/harper-core/src/spell/hunspell/matcher.rs b/harper-core/src/spell/hunspell/matcher.rsindex 299fab9a8..f0574185f 100644--- a/harper-core/src/spell/hunspell/matcher.rs+++ b/harper-core/src/spell/hunspell/matcher.rs@@ -1,10 +1,14 @@+use std::fmt::{Display, Formatter};++use serde::{Deserialize, Serialize};+/// A simplified, Regex-like matcher.////// See Hunspell documentation on affixes for more information.-#[derive(Debug, Clone)]+#[derive(Debug, Clone, Serialize, Deserialize)]pub struct Matcher {/// Position-based operators.- operators: Vec<Operator>+ operators: Vec<Operator>,}impl Matcher {@@ -38,7 +42,7 @@ impl Matcher {}}'.' => operators.push(Operator::Any),- _ => operators.push(Operator::Literal(c))+ _ => operators.push(Operator::Literal(c)),}char_idx += 1;@@ -66,12 +70,43 @@ impl Matcher {}}-#[derive(Debug, Clone, PartialEq, Eq)]+impl Display for Matcher {+ fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {+ for op in &self.operators {+ match op {+ Operator::Literal(c) => write!(f, "{}", c)?,+ Operator::MatchOne(cs) => {+ write!(f, "[")?;++ for c in cs {+ write!(f, "{}", c)?;+ }++ write!(f, "]")?;+ }+ Operator::MatchNone(cs) => {+ write!(f, "[^")?;++ for c in cs {+ write!(f, "{}", c)?;+ }++ write!(f, "]")?;+ }+ Operator::Any => write!(f, ".")?,+ }+ }++ Ok(())+ }+}++#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]enum Operator {Literal(char),MatchOne(Vec<char>),MatchNone(Vec<char>),- Any+ Any,}impl Operator {@@ -80,7 +115,7 @@ impl Operator {Operator::Literal(b) => a == *b,Operator::MatchOne(b) => b.contains(&a),Operator::MatchNone(b) => !b.contains(&a),- Operator::Any => true+ Operator::Any => true,}}}@@ -88,7 +123,7 @@ impl Operator {#[derive(Debug, Clone, Copy, thiserror::Error)]pub enum Error {#[error("Unmatched bracket at index: {index}")]- UnmatchedBracket { index: usize }+ UnmatchedBracket { index: usize },}#[cfg(test)]@@ -120,4 +155,12 @@ mod tests {assert!(matcher.matches(&['o']));assert!(matcher.matches(&['u']));}++ #[test]+ fn round_trip() {+ let source = "[^aeiou]a.s";+ let matcher = Matcher::parse(source).unwrap();++ assert_eq!(matcher.to_string(), source);+ }}diff --git a/harper-core/src/spell/hunspell/mod.rs b/harper-core/src/spell/hunspell/mod.rsindex fd8c5bbbb..8e0ea9ef2 100644--- a/harper-core/src/spell/hunspell/mod.rs+++ b/harper-core/src/spell/hunspell/mod.rs@@ -1,9 +1,11 @@-mod attributes;+mod affix_replacement;+mod attribute_list;mod error;+mod expansion;mod matcher;mod word_list;-pub use attributes::AttributeList;+pub use attribute_list::AttributeList;pub use error::Error;use self::word_list::parse_word_list;@@ -19,7 +21,9 @@ pub fn parse_default_attribute_list() -> Result<AttributeList, Error> {#[cfg(test)]mod tests {- use super::attributes::AttributeList;+ use std::fs;++ use super::attribute_list::AttributeList;use super::word_list::parse_word_list;use super::{parse_default_attribute_list, parse_default_word_list};use crate::CharString;@@ -29,6 +33,15 @@ mod tests {"SET UTF-8\nTRY esianrtolcdugmphbyfvkwzESIANRTOLCDUGMPHBYFVKWZ'\n\nREP 2\nREP f ph\nREP \ph f\n\nPFX A Y 1\nPFX A 0 re .\n\nSFX B Y 2\nSFX B 0 ed [^y]\nSFX B y ied y";+ #[test]+ fn dump() {+ let default = parse_default_attribute_list().unwrap();+ let default_human_readable = default.to_human_readable();++ let dumped = serde_json::to_string_pretty(&default_human_readable).unwrap();+ fs::write("affixes.json", dumped.as_bytes());+ }+#[test]fn correctly_expands_test_files() {let words = parse_word_list(TEST_WORD_LIST).unwrap();@@ -54,7 +67,7 @@ mod tests {let attributes = AttributeList::parse("SFX S Y 4\nSFX S y ies [^aeiou]y\nSFX S 0 s \[aeiou]y\nSFX S 0 es [sxzh]\nSFX S 0 s [^sxzhy]\n\nSFX \- M Y 1\nSFX M 0 's ."+ M Y 1\nSFX M 0 's .",).unwrap();diff --git a/harper-core/src/word_metadata.rs b/harper-core/src/word_metadata.rsnew file mode 100644index 000000000..2f45af7c7--- /dev/null+++ b/harper-core/src/word_metadata.rs@@ -0,0 +1,22 @@+use serde::{Deserialize, Serialize};++#[derive(Debug, Default, Copy, Clone, PartialEq, Eq, Serialize, Deserialize)]+pub struct WordMetadata {+ kind: Option<WordKind>,+ tense: Option<Tense>,+}++#[derive(Debug, Copy, Clone, PartialEq, Eq, Serialize, Deserialize)]+pub enum Tense {+ Past,+ Present,+ Future,+}++#[derive(Debug, Copy, Clone, PartialEq, Eq, Serialize, Deserialize)]+pub enum WordKind {+ Verb,+ Noun { is_proper: bool },+ Adjective,+ Adverb,+}