Skip to content

Offline, privacy-first grammar checker. Fast, open-source, Rust-powered

Small hunspell dictionaries now work

Elijah Potter346a7a7b9dc9
diff --git a/Cargo.lock b/Cargo.lock
index 36d976523..84df93c46 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -622,6 +622,7 @@ dependencies = [
"pulldown-cmark",
"serde",
"smallvec",
+ "thiserror",
]
[[package]]
diff --git a/harper-core/Cargo.toml b/harper-core/Cargo.toml
index 757787608..bc471b3e4 100644
--- a/harper-core/Cargo.toml
+++ b/harper-core/Cargo.toml
@@ -11,6 +11,7 @@ once_cell = "1.19.0"
pulldown-cmark = "0.9.3"
serde = { version = "1.0.190", features = ["derive"] }
smallvec = "1.12.0"
+thiserror = "1.0.56"
[dev-dependencies]
criterion = { version = "0.5.1" }
diff --git a/harper-core/src/linting/wrong_quotes.rs b/harper-core/src/linting/wrong_quotes.rs
index 8a29174bd..3ab512b83 100644
--- a/harper-core/src/linting/wrong_quotes.rs
+++ b/harper-core/src/linting/wrong_quotes.rs
@@ -1,6 +1,5 @@
use crate::{
- document::Document, parsing::Quote, Dictionary, Lint, LintKind, Punctuation, Suggestion, Token,
- TokenKind,
+ document::Document, Dictionary, Lint, LintKind, Punctuation, Suggestion, Token, TokenKind,
};
pub fn wrong_quotes(document: &Document, _dictionary: &Dictionary) -> Vec<Lint> {
diff --git a/harper-core/src/parsing/lexer.rs b/harper-core/src/parsing/lexer.rs
index 285bda6d6..d2b14ba64 100644
--- a/harper-core/src/parsing/lexer.rs
+++ b/harper-core/src/parsing/lexer.rs
@@ -228,7 +228,8 @@ lex_punctuation! {
"]" => CloseSquare,
"(" => OpenRound,
")" => CloseRound,
- "#" => Hash
+ "#" => Hash,
+ "'" => Apostrophe
}
fn lex_quote(source: &[char]) -> Option<FoundToken> {
diff --git a/harper-core/src/parsing/token.rs b/harper-core/src/parsing/token.rs
index d97ff0c08..542fac73c 100644
--- a/harper-core/src/parsing/token.rs
+++ b/harper-core/src/parsing/token.rs
@@ -79,6 +79,8 @@ pub enum Punctuation {
CloseRound,
/// "
Hash,
+ /// '
+ Apostrophe,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
diff --git a/harper-core/src/spell/dictionary.rs b/harper-core/src/spell/dictionary.rs
index f59c5e328..21769f20f 100644
--- a/harper-core/src/spell/dictionary.rs
+++ b/harper-core/src/spell/dictionary.rs
@@ -1,4 +1,4 @@
-use std::{borrow::Borrow, hash::Hasher};
+use std::hash::Hasher;
use ahash::{AHashSet, AHasher};
use once_cell::sync::Lazy;
diff --git a/harper-core/src/spell/hunspell/attributes.rs b/harper-core/src/spell/hunspell/attributes.rs
new file mode 100644
index 000000000..f993ac561
--- /dev/null
+++ b/harper-core/src/spell/hunspell/attributes.rs
@@ -0,0 +1,288 @@
+use itertools::Itertools;
+use std::{mem::replace, usize};
+
+use ahash::HashMap;
+
+use crate::Span;
+
+use super::{matcher::Matcher, word_list::MarkedWord, Error};
+
+#[derive(Debug, Clone)]
+struct AffixReplacement {
+ pub remove: Vec<char>,
+ pub add: Vec<char>,
+ pub condition: Matcher,
+}
+
+#[derive(Debug, Clone)]
+struct Expansion {
+ // If not true, its a prefix
+ pub suffix: bool,
+ pub cross_product: bool,
+ pub replacements: Vec<AffixReplacement>,
+}
+
+#[derive(Debug)]
+pub struct AttributeList {
+ /// Key = Affix Flag
+ affixes: HashMap<char, Expansion>,
+}
+
+impl AttributeList {
+ pub fn parse(file: &str) -> Result<Self, Error> {
+ let mut output = Self {
+ affixes: HashMap::default(),
+ };
+
+ for line in file.lines() {
+ if line.chars().filter(|c| !c.is_whitespace()).count() == 0 {
+ continue;
+ }
+
+ output.parse_line(line)?;
+ }
+
+ Ok(output)
+ }
+
+ fn parse_line(&mut self, line: &str) -> Result<(), Error> {
+ if line.len() < 4 {
+ return Ok(());
+ }
+
+ let mut parser = AttributeArgParser::new(line);
+
+ let suffix = match parser.parse_arg()? {
+ "PFX" => false,
+ "SFX" => true,
+ _ => return Ok(()),
+ };
+
+ let flag = {
+ let flag_arg = parser.parse_arg()?;
+ if flag_arg.len() != 1 {
+ return Err(Error::MultiCharacterFlag);
+ };
+
+ flag_arg.chars().next().unwrap()
+ };
+
+ if let Some(expansion) = self.affixes.get_mut(&flag) {
+ let remove_arg = parser.parse_arg()?;
+
+ let remove: Vec<_> = remove_arg.chars().collect();
+ let remove = if remove.len() == 1 && remove[0] == '0' {
+ vec![]
+ } else {
+ remove
+ };
+
+ let add = parser.parse_arg()?.chars().collect();
+ let condition = Matcher::parse(parser.parse_arg()?)?;
+
+ let replacement = AffixReplacement {
+ remove,
+ add,
+ condition,
+ };
+
+ expansion.replacements.push(replacement)
+ } else {
+ let cross_product = parser.parse_bool_arg()?;
+ dbg!(cross_product);
+ let count = parser.parse_usize_arg()?;
+ dbg!(count);
+
+ self.affixes.insert(
+ flag,
+ Expansion {
+ suffix,
+ cross_product,
+ replacements: Vec::with_capacity(count),
+ },
+ );
+ }
+
+ Ok(())
+ }
+
+ /// Expand [`MarkedWord`] into a list of full words, including itself.
+ ///
+ /// In the future, I want to make this function cleaner and faster.
+ pub fn expand_marked_word(&self, word: MarkedWord) -> Result<Vec<Vec<char>>, Error> {
+ let mut words = Vec::with_capacity(word.attributes.len() + 1);
+
+ for attr in &word.attributes {
+ let expansion = self.affixes.get(attr).ok_or(Error::UndefinedAttribute)?;
+
+ let mut new_words = Vec::new();
+
+ for replacement in &expansion.replacements {
+ new_words.extend(Self::apply_replacement(
+ replacement,
+ &word.letters,
+ expansion.suffix,
+ ))
+ }
+
+ if expansion.cross_product {
+ let mut opp_attr = Vec::new();
+
+ for attr in &word.attributes {
+ let attr_def = self.affixes.get(&attr).ok_or(Error::UndefinedAttribute)?;
+ if attr_def.suffix != expansion.suffix {
+ opp_attr.push(*attr);
+ }
+ }
+
+ let mut cross_product_words = Vec::new();
+
+ for new_word in new_words {
+ cross_product_words.extend(self.expand_marked_word(MarkedWord {
+ letters: new_word,
+ attributes: opp_attr.clone(),
+ })?)
+ }
+
+ words.extend_from_slice(&cross_product_words);
+ } else {
+ words.extend_from_slice(&new_words);
+ }
+ }
+
+ words.push(word.letters);
+
+ Ok(words)
+ }
+
+ pub fn expand_marked_words(
+ &self,
+ words: impl IntoIterator<Item = MarkedWord>,
+ ) -> Result<Vec<Vec<char>>, Error> {
+ let mut output = Vec::new();
+
+ for word in words {
+ output.extend(self.expand_marked_word(word)?.into_iter().unique());
+ }
+
+ Ok(output)
+ }
+
+ fn apply_replacement(
+ replacement: &AffixReplacement,
+ letters: &[char],
+ suffix: bool,
+ ) -> Option<Vec<char>> {
+ if replacement.condition.len() > letters.len() {
+ return None;
+ }
+
+ let target_span = if suffix {
+ Span::new(letters.len() - replacement.condition.len(), letters.len())
+ } else {
+ Span::new(0, replacement.condition.len())
+ };
+
+ let target_segment = target_span.get_content(letters);
+
+ if replacement.condition.matches(target_segment) {
+ let mut replaced_segment = letters.to_vec();
+ let mut remove = replacement.remove.to_vec();
+
+ if !suffix {
+ replaced_segment.reverse();
+ } else {
+ remove.reverse();
+ }
+
+ for c in &remove {
+ let Some(last) = replaced_segment.last() else {
+ return None;
+ };
+
+ if last == c {
+ replaced_segment.pop();
+ } else {
+ return None;
+ }
+ }
+
+ let mut to_add = replacement.add.to_vec();
+
+ if !suffix {
+ to_add.reverse()
+ }
+
+ replaced_segment.extend(to_add);
+
+ if !suffix {
+ replaced_segment.reverse();
+ }
+
+ return Some(replaced_segment);
+ }
+
+ None
+ }
+}
+
+struct AttributeArgParser<'a> {
+ line: &'a str,
+ cursor: usize,
+}
+
+impl<'a> AttributeArgParser<'a> {
+ pub fn new(line: &'a str) -> Self {
+ Self { line, cursor: 0 }
+ }
+
+ // Grap next affix argument, returning an error if it doesn't exist.
+ fn parse_arg(&mut self) -> Result<&'a str, Error> {
+ let Some((next_word_start, _)) = self.line[self.cursor..]
+ .char_indices()
+ .find(|(_i, c)| !c.is_whitespace())
+ else {
+ return Err(Error::UnexpectedEndOfLine);
+ };
+
+ let next_word_end = self.line[self.cursor + next_word_start..]
+ .char_indices()
+ .find(|(_i, c)| c.is_whitespace())
+ .map(|(end, _)| end)
+ .unwrap_or(self.line.len() - self.cursor - next_word_start);
+
+ let abs_start = next_word_start + self.cursor;
+ let abs_end = next_word_start + self.cursor + next_word_end;
+
+ self.cursor = abs_end;
+
+ Ok(&self.line[abs_start..abs_end])
+ }
+
+ // Grab next affix argument, returning an error if it isn't parsable as a number.
+ fn parse_usize_arg(&mut self) -> Result<usize, Error> {
+ self.parse_arg()?
+ .parse()
+ .map_err(|_| Error::ExpectedUnsignedInteger)
+ }
+
+ // Grab next affix argument, returning an error if it isn't Y or N.
+ fn parse_bool_arg(&mut self) -> Result<bool, Error> {
+ match self.parse_arg()? {
+ "Y" => Ok(true),
+ "N" => Ok(false),
+ _ => Err(Error::ExpectedBoolean),
+ }
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::super::tests::ATTR_LIST;
+ use super::AttributeList;
+
+ #[test]
+ fn can_parse_test_file() {
+ AttributeList::parse(ATTR_LIST).unwrap();
+ }
+}
diff --git a/harper-core/src/spell/hunspell/error.rs b/harper-core/src/spell/hunspell/error.rs
new file mode 100644
index 000000000..155567022
--- /dev/null
+++ b/harper-core/src/spell/hunspell/error.rs
@@ -0,0 +1,19 @@
+use super::matcher;
+
+#[derive(Debug, Clone, Copy, thiserror::Error)]
+pub enum Error {
+ #[error("The provided file's item count was malformed.")]
+ MalformedItemCount,
+ #[error("Expected affix flag to be exactly one character.")]
+ MultiCharacterFlag,
+ #[error("Expected affix option to be a boolean.")]
+ ExpectedBoolean,
+ #[error("Expected affix option to be an unsigned integer.")]
+ ExpectedUnsignedInteger,
+ #[error("Could not parse because we encountered the end of the line.")]
+ UnexpectedEndOfLine,
+ #[error("An error occured with a condition: {0}")]
+ MatcherError(#[from] matcher::Error),
+ #[error("An attribute was added to a word that was never defined in the attribute list.")]
+ UndefinedAttribute,
+}
diff --git a/harper-core/src/spell/hunspell/matcher.rs b/harper-core/src/spell/hunspell/matcher.rs
new file mode 100644
index 000000000..14701831e
--- /dev/null
+++ b/harper-core/src/spell/hunspell/matcher.rs
@@ -0,0 +1,124 @@
+/// A simplified, Regex-like matcher.
+///
+/// See Hunspell documentation on affixes for more information.
+#[derive(Debug, Clone)]
+pub struct Matcher {
+ /// Position-based operators.
+ operators: Vec<Operator>,
+}
+
+impl Matcher {
+ pub fn parse(source: &str) -> Result<Self, Error> {
+ let mut operators = Vec::new();
+
+ let char_indices: Vec<_> = source.char_indices().collect();
+ let mut char_idx = 0;
+
+ while char_idx < char_indices.len() {
+ let (idx, c) = char_indices[char_idx];
+
+ match c {
+ '[' => {
+ let close_idx = source[idx..]
+ .find(']')
+ .ok_or(Error::UnmatchedBracket { index: idx })?;
+
+ let bracket_contents = &source[idx + 1..close_idx];
+
+ let invert = matches!(bracket_contents.chars().next(), Some('^'));
+
+ if invert {
+ let chars: Vec<char> = bracket_contents.chars().skip(1).collect();
+ char_idx += chars.len() + 2;
+ operators.push(Operator::MatchNone(chars));
+ } else {
+ let chars: Vec<char> = bracket_contents.chars().collect();
+ char_idx += chars.len() + 1;
+ operators.push(Operator::MatchOne(chars));
+ }
+ }
+ '.' => operators.push(Operator::Any),
+ _ => operators.push(Operator::Literal(c)),
+ }
+
+ char_idx += 1;
+ }
+
+ Ok(Self { operators })
+ }
+
+ pub fn len(&self) -> usize {
+ self.operators.len()
+ }
+
+ pub fn matches(&self, chars: &[char]) -> bool {
+ if chars.len() != self.len() {
+ return false;
+ }
+
+ for (c, op) in chars.into_iter().zip(self.operators.iter()) {
+ if !op.matches(*c) {
+ return false;
+ }
+ }
+
+ true
+ }
+}
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+enum Operator {
+ Literal(char),
+ MatchOne(Vec<char>),
+ MatchNone(Vec<char>),
+ Any,
+}
+
+impl Operator {
+ fn matches(&self, a: char) -> bool {
+ match self {
+ Operator::Literal(b) => a == *b,
+ Operator::MatchOne(b) => b.contains(&a),
+ Operator::MatchNone(b) => !b.contains(&a),
+ Operator::Any => true,
+ }
+ }
+}
+
+#[derive(Debug, Clone, Copy, thiserror::Error)]
+pub enum Error {
+ #[error("Unmatched bracket at index: {index}")]
+ UnmatchedBracket { index: usize },
+}
+
+#[cfg(test)]
+mod tests {
+ use crate::spell::hunspell::matcher::Operator;
+
+ use super::Matcher;
+
+ #[test]
+ fn parses_simple() {
+ let matcher = Matcher::parse("[^aeiou]a.s").unwrap();
+ assert_eq!(
+ matcher.operators,
+ vec![
+ Operator::MatchNone(vec!['a', 'e', 'i', 'o', 'u']),
+ Operator::Literal('a'),
+ Operator::Any,
+ Operator::Literal('s')
+ ]
+ )
+ }
+
+ #[test]
+ fn matches_vowels() {
+ let matcher = Matcher::parse("[aeiou]").unwrap();
+
+ assert!(matcher.matches(&['a']));
+ assert!(matcher.matches(&['e']));
+ assert!(matcher.matches(&['i']));
+ assert!(matcher.matches(&['o']));
+ assert!(matcher.matches(&['u']));
+ }
+}
diff --git a/harper-core/src/spell/hunspell/mod.rs b/harper-core/src/spell/hunspell/mod.rs
new file mode 100644
index 000000000..4831d1982
--- /dev/null
+++ b/harper-core/src/spell/hunspell/mod.rs
@@ -0,0 +1,33 @@
+mod attributes;
+mod error;
+mod matcher;
+mod word_list;
+
+pub use error::Error;
+
+#[cfg(test)]
+mod tests {
+ use super::{attributes::AttributeList, word_list::parse_word_list};
+
+ pub const TEST_WORD_LIST: &str = "3\nhello\ntry/B\nwork/AB";
+ pub const ATTR_LIST: &str = "SET UTF-8\nTRY esianrtolcdugmphbyfvkwzESIANRTOLCDUGMPHBYFVKWZ'\n\nREP 2\nREP f ph\nREP ph f\n\nPFX A Y 1\nPFX A 0 re .\n\nSFX B Y 2\nSFX B 0 ed [^y]\nSFX B y ied y";
+
+ #[test]
+ fn correctly_expands_test_files() {
+ let words = parse_word_list(TEST_WORD_LIST).unwrap();
+ let attributes = AttributeList::parse(ATTR_LIST).unwrap();
+
+ dbg!(&attributes);
+
+ let expanded = attributes.expand_marked_words(words).unwrap();
+ let expanded: Vec<String> = expanded
+ .into_iter()
+ .map(|v| v.into_iter().collect())
+ .collect();
+
+ assert_eq!(
+ expanded,
+ vec!["hello", "tried", "try", "reworked", "rework", "worked", "work"]
+ )
+ }
+}
diff --git a/harper-core/src/spell/hunspell/word_list.rs b/harper-core/src/spell/hunspell/word_list.rs
new file mode 100644
index 000000000..9bce1021d
--- /dev/null
+++ b/harper-core/src/spell/hunspell/word_list.rs
@@ -0,0 +1,55 @@
+use super::Error;
+
+pub struct MarkedWord {
+ pub letters: Vec<char>,
+ pub attributes: Vec<char>,
+}
+
+pub struct Dictionary {
+ pub(self) words: Vec<MarkedWord>,
+}
+
+/// Parse a hunspell word list
+///
+/// Returns [`None`] if the given string is invalid.
+pub fn parse_word_list(source: &str) -> Result<Vec<MarkedWord>, Error> {
+ let mut lines = source.lines();
+
+ let approx_item_count = lines
+ .next()
+ .ok_or(Error::MalformedItemCount)?
+ .parse()
+ .map_err(|_| Error::MalformedItemCount)?;
+
+ let mut words = Vec::with_capacity(approx_item_count);
+
+ for line in lines {
+ if let Some((word, attributes)) = line.split_once('/') {
+ words.push(MarkedWord {
+ letters: word.chars().collect(),
+ attributes: attributes.chars().collect(),
+ })
+ } else {
+ words.push(MarkedWord {
+ letters: line.chars().collect(),
+ attributes: Vec::new(),
+ })
+ }
+ }
+
+ Ok(words)
+}
+
+#[cfg(test)]
+mod tests {
+ use super::super::tests::TEST_WORD_LIST;
+ use super::parse_word_list;
+
+ #[test]
+ fn can_parse_test_file() {
+ let list = parse_word_list(TEST_WORD_LIST).unwrap();
+
+ assert_eq!(list.last().unwrap().attributes.len(), 2);
+ assert_eq!(list.len(), 3);
+ }
+}
diff --git a/harper-core/src/spell/mod.rs b/harper-core/src/spell/mod.rs
index c74766baf..cf18aa1f7 100644
--- a/harper-core/src/spell/mod.rs
+++ b/harper-core/src/spell/mod.rs
@@ -1,6 +1,7 @@
pub use self::dictionary::Dictionary;
mod dictionary;
+mod hunspell;
/// Suggest a correct spelling for a given misspelled word.
/// [misspelled_word] is assumed to be quite small (n < 100)