Skip to content

Offline, privacy-first grammar checker. Fast, open-source, Rust-powered

feat: wrote conversion from hunspell dictionary to custom format

Elijah Potter1d9ae5e20883
diff --git a/Cargo.lock b/Cargo.lock
index ee23ec475..ac95a7237 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -538,6 +538,7 @@ dependencies = [
"pulldown-cmark",
"rand",
"serde",
+ "serde_json",
"smallvec",
"thiserror",
"unicode-blocks",
@@ -1092,11 +1093,12 @@ dependencies = [
[[package]]
name = "serde_json"
-version = "1.0.117"
+version = "1.0.121"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "455182ea6142b14f93f4bc5320a2b31c1f266b66a4a5c858b013302a5d8cbfc3"
+checksum = "4ab380d7d9f22ef3f21ad3e6c1ebe8e4fc7a2000ccba2e4d71fc96f15b2cb609"
dependencies = [
"itoa",
+ "memchr",
"ryu",
"serde",
]
diff --git a/harper-core/Cargo.toml b/harper-core/Cargo.toml
index b2664c0d1..ac9b68212 100644
--- a/harper-core/Cargo.toml
+++ b/harper-core/Cargo.toml
@@ -16,6 +16,7 @@ once_cell = "1.19.0"
paste = "1.0.14"
pulldown-cmark = "0.11.0"
serde = { version = "1.0.197", features = ["derive"] }
+serde_json = "1.0.121"
smallvec = { version = "1.13.2", features = ["serde"] }
thiserror = "1.0.57"
unicode-blocks = "0.1.9"
diff --git a/harper-core/src/lib.rs b/harper-core/src/lib.rs
index d69e87d8e..fc7be0206 100644
--- a/harper-core/src/lib.rs
+++ b/harper-core/src/lib.rs
@@ -15,6 +15,7 @@ mod spell;
mod sync;
mod token;
mod vec_ext;
+mod word_metadata;
pub use char_string::{CharString, CharStringExt};
pub use document::Document;
@@ -26,6 +27,7 @@ pub use spell::{Dictionary, FullDictionary, MergedDictionary};
pub use sync::Lrc;
pub use token::{FatToken, Token, TokenKind, TokenStringExt};
pub use vec_ext::VecExt;
+pub use word_metadata::{Tense, WordKind, WordMetadata};
/// A utility function that removes overlapping lints in a vector,
/// keeping the more important ones.
diff --git a/harper-core/src/spell/hunspell/affix_replacement.rs b/harper-core/src/spell/hunspell/affix_replacement.rs
new file mode 100644
index 000000000..17744a590
--- /dev/null
+++ b/harper-core/src/spell/hunspell/affix_replacement.rs
@@ -0,0 +1,39 @@
+use serde::{Deserialize, Serialize};
+
+use super::{matcher::Matcher, Error};
+
+#[derive(Debug, Clone)]
+pub struct AffixReplacement {
+ pub remove: Vec<char>,
+ pub add: Vec<char>,
+ pub condition: Matcher,
+}
+
+impl AffixReplacement {
+ pub fn to_human_readable(&self) -> HumanReadableAffixReplacement {
+ HumanReadableAffixReplacement {
+ remove: self.remove.iter().collect(),
+ add: self.add.iter().collect(),
+ condition: self.condition.to_string(),
+ }
+ }
+}
+
+/// A version of [`AffixReplacement`] that can be serialized to JSON (or whatever) and maintain the
+/// nice Regex syntax of the inner [`Matcher`].
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct HumanReadableAffixReplacement {
+ pub remove: String,
+ pub add: String,
+ pub condition: String,
+}
+
+impl HumanReadableAffixReplacement {
+ pub fn to_normal(&self) -> Result<AffixReplacement, Error> {
+ Ok(AffixReplacement {
+ remove: self.remove.chars().collect(),
+ add: self.add.chars().collect(),
+ condition: Matcher::parse(&self.condition)?,
+ })
+ }
+}
diff --git a/harper-core/src/spell/hunspell/attributes.rs b/harper-core/src/spell/hunspell/attribute_list.rs
similarity index 82%
rename from harper-core/src/spell/hunspell/attributes.rs
rename to harper-core/src/spell/hunspell/attribute_list.rs
index f89886798..7b521b0f9 100644
--- a/harper-core/src/spell/hunspell/attributes.rs
+++ b/harper-core/src/spell/hunspell/attribute_list.rs
@@ -1,36 +1,23 @@
use hashbrown::HashMap;
+use serde::{Deserialize, Serialize};
use smallvec::ToSmallVec;
-use super::matcher::Matcher;
+use super::expansion::{Expansion, HumanReadableExpansion};
use super::word_list::MarkedWord;
use super::Error;
-use crate::{CharString, Span};
-
-#[derive(Debug, Clone)]
-struct AffixReplacement {
- pub remove: Vec<char>,
- pub add: Vec<char>,
- pub condition: Matcher
-}
-
-#[derive(Debug, Clone)]
-struct Expansion {
- // If not true, its a prefix
- pub suffix: bool,
- pub cross_product: bool,
- pub replacements: Vec<AffixReplacement>
-}
+use super::{affix_replacement::AffixReplacement, matcher::Matcher};
+use crate::{CharString, Span, WordMetadata};
#[derive(Debug, Clone)]
pub struct AttributeList {
/// Key = Affix Flag
- affixes: HashMap<char, Expansion>
+ affixes: HashMap<char, Expansion>,
}
impl AttributeList {
pub fn parse(file: &str) -> Result<Self, Error> {
let mut output = Self {
- affixes: HashMap::default()
+ affixes: HashMap::default(),
};
for line in file.lines() {
@@ -44,6 +31,16 @@ impl AttributeList {
Ok(output)
}
+ pub fn to_human_readable(&self) -> HumanReadableAttributeList {
+ HumanReadableAttributeList {
+ affixes: self
+ .affixes
+ .iter()
+ .map(|(affix, exp)| (*affix, exp.to_human_readable()))
+ .collect(),
+ }
+ }
+
fn parse_line(&mut self, line: &str) -> Result<(), Error> {
if line.len() < 4 {
return Ok(());
@@ -54,7 +51,7 @@ impl AttributeList {
let suffix = match parser.parse_arg()? {
"PFX" => false,
"SFX" => true,
- _ => return Ok(())
+ _ => return Ok(()),
};
let flag = {
@@ -82,7 +79,7 @@ impl AttributeList {
let replacement = AffixReplacement {
remove,
add,
- condition
+ condition,
};
expansion.replacements.push(replacement)
@@ -95,8 +92,9 @@ impl AttributeList {
Expansion {
suffix,
cross_product,
- replacements: Vec::with_capacity(count)
- }
+ replacements: Vec::with_capacity(count),
+ adds_metadata: WordMetadata::default(),
+ },
);
}
@@ -124,7 +122,7 @@ impl AttributeList {
new_words.extend(Self::apply_replacement(
replacement,
&word.letters,
- expansion.suffix
+ expansion.suffix,
))
}
@@ -146,9 +144,9 @@ impl AttributeList {
self.expand_marked_word(
MarkedWord {
letters: new_word,
- attributes: opp_attr.clone()
+ attributes: opp_attr.clone(),
},
- dest
+ dest,
);
}
@@ -173,7 +171,7 @@ impl AttributeList {
pub fn expand_marked_words(
&self,
words: impl IntoIterator<Item = MarkedWord>,
- dest: &mut Vec<CharString>
+ dest: &mut Vec<CharString>,
) {
for word in words {
self.expand_marked_word(word, dest);
@@ -183,7 +181,7 @@ impl AttributeList {
fn apply_replacement(
replacement: &AffixReplacement,
letters: &[char],
- suffix: bool
+ suffix: bool,
) -> Option<CharString> {
if replacement.condition.len() > letters.len() {
return None;
@@ -236,9 +234,26 @@ impl AttributeList {
}
}
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct HumanReadableAttributeList {
+ affixes: HashMap<char, HumanReadableExpansion>,
+}
+
+impl HumanReadableAttributeList {
+ pub fn to_normal(&self) -> Result<AttributeList, Error> {
+ let mut affixes = HashMap::with_capacity(self.affixes.len());
+
+ for (affix, expansion) in &self.affixes {
+ affixes.insert(*affix, expansion.to_normal()?);
+ }
+
+ Ok(AttributeList { affixes })
+ }
+}
+
struct AttributeArgParser<'a> {
line: &'a str,
- cursor: usize
+ cursor: usize,
}
impl<'a> AttributeArgParser<'a> {
@@ -246,7 +261,7 @@ impl<'a> AttributeArgParser<'a> {
Self { line, cursor: 0 }
}
- // Grap next affix argument, returning an error if it doesn't exist.
+ // Grab next affix argument, returning an error if it doesn't exist.
fn parse_arg(&mut self) -> Result<&'a str, Error> {
let Some((next_word_start, _)) = self.line[self.cursor..]
.char_indices()
@@ -282,7 +297,7 @@ impl<'a> AttributeArgParser<'a> {
match self.parse_arg()? {
"Y" => Ok(true),
"N" => Ok(false),
- _ => Err(Error::ExpectedBoolean)
+ _ => Err(Error::ExpectedBoolean),
}
}
}
diff --git a/harper-core/src/spell/hunspell/expansion.rs b/harper-core/src/spell/hunspell/expansion.rs
new file mode 100644
index 000000000..21a64fdf6
--- /dev/null
+++ b/harper-core/src/spell/hunspell/expansion.rs
@@ -0,0 +1,57 @@
+use serde::{Deserialize, Serialize};
+
+use crate::WordMetadata;
+
+use super::{
+ affix_replacement::{AffixReplacement, HumanReadableAffixReplacement},
+ Error,
+};
+
+#[derive(Debug, Clone)]
+pub struct Expansion {
+ /// If `!true`, this is a prefix
+ pub suffix: bool,
+ pub cross_product: bool,
+ pub replacements: Vec<AffixReplacement>,
+ pub adds_metadata: WordMetadata,
+}
+
+impl Expansion {
+ pub fn to_human_readable(&self) -> HumanReadableExpansion {
+ HumanReadableExpansion {
+ suffix: self.suffix,
+ cross_product: self.cross_product,
+ replacements: self
+ .replacements
+ .iter()
+ .map(AffixReplacement::to_human_readable)
+ .collect(),
+ adds_metadata: self.adds_metadata,
+ }
+ }
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct HumanReadableExpansion {
+ pub suffix: bool,
+ pub cross_product: bool,
+ pub replacements: Vec<HumanReadableAffixReplacement>,
+ pub adds_metadata: WordMetadata,
+}
+
+impl HumanReadableExpansion {
+ pub fn to_normal(&self) -> Result<Expansion, Error> {
+ let mut replacements = Vec::with_capacity(self.replacements.len());
+
+ for replacement in &self.replacements {
+ replacements.push(replacement.to_normal()?);
+ }
+
+ Ok(Expansion {
+ suffix: self.suffix,
+ cross_product: self.cross_product,
+ replacements,
+ adds_metadata: self.adds_metadata,
+ })
+ }
+}
diff --git a/harper-core/src/spell/hunspell/matcher.rs b/harper-core/src/spell/hunspell/matcher.rs
index 299fab9a8..f0574185f 100644
--- a/harper-core/src/spell/hunspell/matcher.rs
+++ b/harper-core/src/spell/hunspell/matcher.rs
@@ -1,10 +1,14 @@
+use std::fmt::{Display, Formatter};
+
+use serde::{Deserialize, Serialize};
+
/// A simplified, Regex-like matcher.
///
/// See Hunspell documentation on affixes for more information.
-#[derive(Debug, Clone)]
+#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct Matcher {
/// Position-based operators.
- operators: Vec<Operator>
+ operators: Vec<Operator>,
}
impl Matcher {
@@ -38,7 +42,7 @@ impl Matcher {
}
}
'.' => operators.push(Operator::Any),
- _ => operators.push(Operator::Literal(c))
+ _ => operators.push(Operator::Literal(c)),
}
char_idx += 1;
@@ -66,12 +70,43 @@ impl Matcher {
}
}
-#[derive(Debug, Clone, PartialEq, Eq)]
+impl Display for Matcher {
+ fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+ for op in &self.operators {
+ match op {
+ Operator::Literal(c) => write!(f, "{}", c)?,
+ Operator::MatchOne(cs) => {
+ write!(f, "[")?;
+
+ for c in cs {
+ write!(f, "{}", c)?;
+ }
+
+ write!(f, "]")?;
+ }
+ Operator::MatchNone(cs) => {
+ write!(f, "[^")?;
+
+ for c in cs {
+ write!(f, "{}", c)?;
+ }
+
+ write!(f, "]")?;
+ }
+ Operator::Any => write!(f, ".")?,
+ }
+ }
+
+ Ok(())
+ }
+}
+
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
enum Operator {
Literal(char),
MatchOne(Vec<char>),
MatchNone(Vec<char>),
- Any
+ Any,
}
impl Operator {
@@ -80,7 +115,7 @@ impl Operator {
Operator::Literal(b) => a == *b,
Operator::MatchOne(b) => b.contains(&a),
Operator::MatchNone(b) => !b.contains(&a),
- Operator::Any => true
+ Operator::Any => true,
}
}
}
@@ -88,7 +123,7 @@ impl Operator {
#[derive(Debug, Clone, Copy, thiserror::Error)]
pub enum Error {
#[error("Unmatched bracket at index: {index}")]
- UnmatchedBracket { index: usize }
+ UnmatchedBracket { index: usize },
}
#[cfg(test)]
@@ -120,4 +155,12 @@ mod tests {
assert!(matcher.matches(&['o']));
assert!(matcher.matches(&['u']));
}
+
+ #[test]
+ fn round_trip() {
+ let source = "[^aeiou]a.s";
+ let matcher = Matcher::parse(source).unwrap();
+
+ assert_eq!(matcher.to_string(), source);
+ }
}
diff --git a/harper-core/src/spell/hunspell/mod.rs b/harper-core/src/spell/hunspell/mod.rs
index fd8c5bbbb..8e0ea9ef2 100644
--- a/harper-core/src/spell/hunspell/mod.rs
+++ b/harper-core/src/spell/hunspell/mod.rs
@@ -1,9 +1,11 @@
-mod attributes;
+mod affix_replacement;
+mod attribute_list;
mod error;
+mod expansion;
mod matcher;
mod word_list;
-pub use attributes::AttributeList;
+pub use attribute_list::AttributeList;
pub use error::Error;
use self::word_list::parse_word_list;
@@ -19,7 +21,9 @@ pub fn parse_default_attribute_list() -> Result<AttributeList, Error> {
#[cfg(test)]
mod tests {
- use super::attributes::AttributeList;
+ use std::fs;
+
+ use super::attribute_list::AttributeList;
use super::word_list::parse_word_list;
use super::{parse_default_attribute_list, parse_default_word_list};
use crate::CharString;
@@ -29,6 +33,15 @@ mod tests {
"SET UTF-8\nTRY esianrtolcdugmphbyfvkwzESIANRTOLCDUGMPHBYFVKWZ'\n\nREP 2\nREP f ph\nREP \
ph f\n\nPFX A Y 1\nPFX A 0 re .\n\nSFX B Y 2\nSFX B 0 ed [^y]\nSFX B y ied y";
+ #[test]
+ fn dump() {
+ let default = parse_default_attribute_list().unwrap();
+ let default_human_readable = default.to_human_readable();
+
+ let dumped = serde_json::to_string_pretty(&default_human_readable).unwrap();
+ fs::write("affixes.json", dumped.as_bytes());
+ }
+
#[test]
fn correctly_expands_test_files() {
let words = parse_word_list(TEST_WORD_LIST).unwrap();
@@ -54,7 +67,7 @@ mod tests {
let attributes = AttributeList::parse(
"SFX S Y 4\nSFX S y ies [^aeiou]y\nSFX S 0 s \
[aeiou]y\nSFX S 0 es [sxzh]\nSFX S 0 s [^sxzhy]\n\nSFX \
- M Y 1\nSFX M 0 's ."
+ M Y 1\nSFX M 0 's .",
)
.unwrap();
diff --git a/harper-core/src/word_metadata.rs b/harper-core/src/word_metadata.rs
new file mode 100644
index 000000000..2f45af7c7
--- /dev/null
+++ b/harper-core/src/word_metadata.rs
@@ -0,0 +1,22 @@
+use serde::{Deserialize, Serialize};
+
+#[derive(Debug, Default, Copy, Clone, PartialEq, Eq, Serialize, Deserialize)]
+pub struct WordMetadata {
+ kind: Option<WordKind>,
+ tense: Option<Tense>,
+}
+
+#[derive(Debug, Copy, Clone, PartialEq, Eq, Serialize, Deserialize)]
+pub enum Tense {
+ Past,
+ Present,
+ Future,
+}
+
+#[derive(Debug, Copy, Clone, PartialEq, Eq, Serialize, Deserialize)]
+pub enum WordKind {
+ Verb,
+ Noun { is_proper: bool },
+ Adjective,
+ Adverb,
+}