Offline, privacy-first grammar checker. Fast, open-source, Rust-powered
Small hunspell dictionaries now work
Elijah Potter346a7a7b9dc9
diff --git a/Cargo.lock b/Cargo.lockindex 36d976523..84df93c46 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -622,6 +622,7 @@ dependencies = ["pulldown-cmark","serde","smallvec",+ "thiserror",][[package]]diff --git a/harper-core/Cargo.toml b/harper-core/Cargo.tomlindex 757787608..bc471b3e4 100644--- a/harper-core/Cargo.toml+++ b/harper-core/Cargo.toml@@ -11,6 +11,7 @@ once_cell = "1.19.0"pulldown-cmark = "0.9.3"serde = { version = "1.0.190", features = ["derive"] }smallvec = "1.12.0"+thiserror = "1.0.56"[dev-dependencies]criterion = { version = "0.5.1" }diff --git a/harper-core/src/linting/wrong_quotes.rs b/harper-core/src/linting/wrong_quotes.rsindex 8a29174bd..3ab512b83 100644--- a/harper-core/src/linting/wrong_quotes.rs+++ b/harper-core/src/linting/wrong_quotes.rs@@ -1,6 +1,5 @@use crate::{- document::Document, parsing::Quote, Dictionary, Lint, LintKind, Punctuation, Suggestion, Token,- TokenKind,+ document::Document, Dictionary, Lint, LintKind, Punctuation, Suggestion, Token, TokenKind,};pub fn wrong_quotes(document: &Document, _dictionary: &Dictionary) -> Vec<Lint> {diff --git a/harper-core/src/parsing/lexer.rs b/harper-core/src/parsing/lexer.rsindex 285bda6d6..d2b14ba64 100644--- a/harper-core/src/parsing/lexer.rs+++ b/harper-core/src/parsing/lexer.rs@@ -228,7 +228,8 @@ lex_punctuation! {"]" => CloseSquare,"(" => OpenRound,")" => CloseRound,- "#" => Hash+ "#" => Hash,+ "'" => Apostrophe}fn lex_quote(source: &[char]) -> Option<FoundToken> {diff --git a/harper-core/src/parsing/token.rs b/harper-core/src/parsing/token.rsindex d97ff0c08..542fac73c 100644--- a/harper-core/src/parsing/token.rs+++ b/harper-core/src/parsing/token.rs@@ -79,6 +79,8 @@ pub enum Punctuation {CloseRound,/// "Hash,+ /// '+ Apostrophe,}#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]diff --git a/harper-core/src/spell/dictionary.rs b/harper-core/src/spell/dictionary.rsindex f59c5e328..21769f20f 100644--- a/harper-core/src/spell/dictionary.rs+++ b/harper-core/src/spell/dictionary.rs@@ -1,4 +1,4 @@-use std::{borrow::Borrow, hash::Hasher};+use std::hash::Hasher;use ahash::{AHashSet, AHasher};use once_cell::sync::Lazy;diff --git a/harper-core/src/spell/hunspell/attributes.rs b/harper-core/src/spell/hunspell/attributes.rsnew file mode 100644index 000000000..f993ac561--- /dev/null+++ b/harper-core/src/spell/hunspell/attributes.rs@@ -0,0 +1,288 @@+use itertools::Itertools;+use std::{mem::replace, usize};++use ahash::HashMap;++use crate::Span;++use super::{matcher::Matcher, word_list::MarkedWord, Error};++#[derive(Debug, Clone)]+struct AffixReplacement {+ pub remove: Vec<char>,+ pub add: Vec<char>,+ pub condition: Matcher,+}++#[derive(Debug, Clone)]+struct Expansion {+ // If not true, its a prefix+ pub suffix: bool,+ pub cross_product: bool,+ pub replacements: Vec<AffixReplacement>,+}++#[derive(Debug)]+pub struct AttributeList {+ /// Key = Affix Flag+ affixes: HashMap<char, Expansion>,+}++impl AttributeList {+ pub fn parse(file: &str) -> Result<Self, Error> {+ let mut output = Self {+ affixes: HashMap::default(),+ };++ for line in file.lines() {+ if line.chars().filter(|c| !c.is_whitespace()).count() == 0 {+ continue;+ }++ output.parse_line(line)?;+ }++ Ok(output)+ }++ fn parse_line(&mut self, line: &str) -> Result<(), Error> {+ if line.len() < 4 {+ return Ok(());+ }++ let mut parser = AttributeArgParser::new(line);++ let suffix = match parser.parse_arg()? {+ "PFX" => false,+ "SFX" => true,+ _ => return Ok(()),+ };++ let flag = {+ let flag_arg = parser.parse_arg()?;+ if flag_arg.len() != 1 {+ return Err(Error::MultiCharacterFlag);+ };++ flag_arg.chars().next().unwrap()+ };++ if let Some(expansion) = self.affixes.get_mut(&flag) {+ let remove_arg = parser.parse_arg()?;++ let remove: Vec<_> = remove_arg.chars().collect();+ let remove = if remove.len() == 1 && remove[0] == '0' {+ vec![]+ } else {+ remove+ };++ let add = parser.parse_arg()?.chars().collect();+ let condition = Matcher::parse(parser.parse_arg()?)?;++ let replacement = AffixReplacement {+ remove,+ add,+ condition,+ };++ expansion.replacements.push(replacement)+ } else {+ let cross_product = parser.parse_bool_arg()?;+ dbg!(cross_product);+ let count = parser.parse_usize_arg()?;+ dbg!(count);++ self.affixes.insert(+ flag,+ Expansion {+ suffix,+ cross_product,+ replacements: Vec::with_capacity(count),+ },+ );+ }++ Ok(())+ }++ /// Expand [`MarkedWord`] into a list of full words, including itself.+ ///+ /// In the future, I want to make this function cleaner and faster.+ pub fn expand_marked_word(&self, word: MarkedWord) -> Result<Vec<Vec<char>>, Error> {+ let mut words = Vec::with_capacity(word.attributes.len() + 1);++ for attr in &word.attributes {+ let expansion = self.affixes.get(attr).ok_or(Error::UndefinedAttribute)?;++ let mut new_words = Vec::new();++ for replacement in &expansion.replacements {+ new_words.extend(Self::apply_replacement(+ replacement,+ &word.letters,+ expansion.suffix,+ ))+ }++ if expansion.cross_product {+ let mut opp_attr = Vec::new();++ for attr in &word.attributes {+ let attr_def = self.affixes.get(&attr).ok_or(Error::UndefinedAttribute)?;+ if attr_def.suffix != expansion.suffix {+ opp_attr.push(*attr);+ }+ }++ let mut cross_product_words = Vec::new();++ for new_word in new_words {+ cross_product_words.extend(self.expand_marked_word(MarkedWord {+ letters: new_word,+ attributes: opp_attr.clone(),+ })?)+ }++ words.extend_from_slice(&cross_product_words);+ } else {+ words.extend_from_slice(&new_words);+ }+ }++ words.push(word.letters);++ Ok(words)+ }++ pub fn expand_marked_words(+ &self,+ words: impl IntoIterator<Item = MarkedWord>,+ ) -> Result<Vec<Vec<char>>, Error> {+ let mut output = Vec::new();++ for word in words {+ output.extend(self.expand_marked_word(word)?.into_iter().unique());+ }++ Ok(output)+ }++ fn apply_replacement(+ replacement: &AffixReplacement,+ letters: &[char],+ suffix: bool,+ ) -> Option<Vec<char>> {+ if replacement.condition.len() > letters.len() {+ return None;+ }++ let target_span = if suffix {+ Span::new(letters.len() - replacement.condition.len(), letters.len())+ } else {+ Span::new(0, replacement.condition.len())+ };++ let target_segment = target_span.get_content(letters);++ if replacement.condition.matches(target_segment) {+ let mut replaced_segment = letters.to_vec();+ let mut remove = replacement.remove.to_vec();++ if !suffix {+ replaced_segment.reverse();+ } else {+ remove.reverse();+ }++ for c in &remove {+ let Some(last) = replaced_segment.last() else {+ return None;+ };++ if last == c {+ replaced_segment.pop();+ } else {+ return None;+ }+ }++ let mut to_add = replacement.add.to_vec();++ if !suffix {+ to_add.reverse()+ }++ replaced_segment.extend(to_add);++ if !suffix {+ replaced_segment.reverse();+ }++ return Some(replaced_segment);+ }++ None+ }+}++struct AttributeArgParser<'a> {+ line: &'a str,+ cursor: usize,+}++impl<'a> AttributeArgParser<'a> {+ pub fn new(line: &'a str) -> Self {+ Self { line, cursor: 0 }+ }++ // Grap next affix argument, returning an error if it doesn't exist.+ fn parse_arg(&mut self) -> Result<&'a str, Error> {+ let Some((next_word_start, _)) = self.line[self.cursor..]+ .char_indices()+ .find(|(_i, c)| !c.is_whitespace())+ else {+ return Err(Error::UnexpectedEndOfLine);+ };++ let next_word_end = self.line[self.cursor + next_word_start..]+ .char_indices()+ .find(|(_i, c)| c.is_whitespace())+ .map(|(end, _)| end)+ .unwrap_or(self.line.len() - self.cursor - next_word_start);++ let abs_start = next_word_start + self.cursor;+ let abs_end = next_word_start + self.cursor + next_word_end;++ self.cursor = abs_end;++ Ok(&self.line[abs_start..abs_end])+ }++ // Grab next affix argument, returning an error if it isn't parsable as a number.+ fn parse_usize_arg(&mut self) -> Result<usize, Error> {+ self.parse_arg()?+ .parse()+ .map_err(|_| Error::ExpectedUnsignedInteger)+ }++ // Grab next affix argument, returning an error if it isn't Y or N.+ fn parse_bool_arg(&mut self) -> Result<bool, Error> {+ match self.parse_arg()? {+ "Y" => Ok(true),+ "N" => Ok(false),+ _ => Err(Error::ExpectedBoolean),+ }+ }+}++#[cfg(test)]+mod tests {+ use super::super::tests::ATTR_LIST;+ use super::AttributeList;++ #[test]+ fn can_parse_test_file() {+ AttributeList::parse(ATTR_LIST).unwrap();+ }+}diff --git a/harper-core/src/spell/hunspell/error.rs b/harper-core/src/spell/hunspell/error.rsnew file mode 100644index 000000000..155567022--- /dev/null+++ b/harper-core/src/spell/hunspell/error.rs@@ -0,0 +1,19 @@+use super::matcher;++#[derive(Debug, Clone, Copy, thiserror::Error)]+pub enum Error {+ #[error("The provided file's item count was malformed.")]+ MalformedItemCount,+ #[error("Expected affix flag to be exactly one character.")]+ MultiCharacterFlag,+ #[error("Expected affix option to be a boolean.")]+ ExpectedBoolean,+ #[error("Expected affix option to be an unsigned integer.")]+ ExpectedUnsignedInteger,+ #[error("Could not parse because we encountered the end of the line.")]+ UnexpectedEndOfLine,+ #[error("An error occured with a condition: {0}")]+ MatcherError(#[from] matcher::Error),+ #[error("An attribute was added to a word that was never defined in the attribute list.")]+ UndefinedAttribute,+}diff --git a/harper-core/src/spell/hunspell/matcher.rs b/harper-core/src/spell/hunspell/matcher.rsnew file mode 100644index 000000000..14701831e--- /dev/null+++ b/harper-core/src/spell/hunspell/matcher.rs@@ -0,0 +1,124 @@+/// A simplified, Regex-like matcher.+///+/// See Hunspell documentation on affixes for more information.+#[derive(Debug, Clone)]+pub struct Matcher {+ /// Position-based operators.+ operators: Vec<Operator>,+}++impl Matcher {+ pub fn parse(source: &str) -> Result<Self, Error> {+ let mut operators = Vec::new();++ let char_indices: Vec<_> = source.char_indices().collect();+ let mut char_idx = 0;++ while char_idx < char_indices.len() {+ let (idx, c) = char_indices[char_idx];++ match c {+ '[' => {+ let close_idx = source[idx..]+ .find(']')+ .ok_or(Error::UnmatchedBracket { index: idx })?;++ let bracket_contents = &source[idx + 1..close_idx];++ let invert = matches!(bracket_contents.chars().next(), Some('^'));++ if invert {+ let chars: Vec<char> = bracket_contents.chars().skip(1).collect();+ char_idx += chars.len() + 2;+ operators.push(Operator::MatchNone(chars));+ } else {+ let chars: Vec<char> = bracket_contents.chars().collect();+ char_idx += chars.len() + 1;+ operators.push(Operator::MatchOne(chars));+ }+ }+ '.' => operators.push(Operator::Any),+ _ => operators.push(Operator::Literal(c)),+ }++ char_idx += 1;+ }++ Ok(Self { operators })+ }++ pub fn len(&self) -> usize {+ self.operators.len()+ }++ pub fn matches(&self, chars: &[char]) -> bool {+ if chars.len() != self.len() {+ return false;+ }++ for (c, op) in chars.into_iter().zip(self.operators.iter()) {+ if !op.matches(*c) {+ return false;+ }+ }++ true+ }+}++#[derive(Debug, Clone, PartialEq, Eq)]+enum Operator {+ Literal(char),+ MatchOne(Vec<char>),+ MatchNone(Vec<char>),+ Any,+}++impl Operator {+ fn matches(&self, a: char) -> bool {+ match self {+ Operator::Literal(b) => a == *b,+ Operator::MatchOne(b) => b.contains(&a),+ Operator::MatchNone(b) => !b.contains(&a),+ Operator::Any => true,+ }+ }+}++#[derive(Debug, Clone, Copy, thiserror::Error)]+pub enum Error {+ #[error("Unmatched bracket at index: {index}")]+ UnmatchedBracket { index: usize },+}++#[cfg(test)]+mod tests {+ use crate::spell::hunspell::matcher::Operator;++ use super::Matcher;++ #[test]+ fn parses_simple() {+ let matcher = Matcher::parse("[^aeiou]a.s").unwrap();+ assert_eq!(+ matcher.operators,+ vec![+ Operator::MatchNone(vec!['a', 'e', 'i', 'o', 'u']),+ Operator::Literal('a'),+ Operator::Any,+ Operator::Literal('s')+ ]+ )+ }++ #[test]+ fn matches_vowels() {+ let matcher = Matcher::parse("[aeiou]").unwrap();++ assert!(matcher.matches(&['a']));+ assert!(matcher.matches(&['e']));+ assert!(matcher.matches(&['i']));+ assert!(matcher.matches(&['o']));+ assert!(matcher.matches(&['u']));+ }+}diff --git a/harper-core/src/spell/hunspell/mod.rs b/harper-core/src/spell/hunspell/mod.rsnew file mode 100644index 000000000..4831d1982--- /dev/null+++ b/harper-core/src/spell/hunspell/mod.rs@@ -0,0 +1,33 @@+mod attributes;+mod error;+mod matcher;+mod word_list;++pub use error::Error;++#[cfg(test)]+mod tests {+ use super::{attributes::AttributeList, word_list::parse_word_list};++ pub const TEST_WORD_LIST: &str = "3\nhello\ntry/B\nwork/AB";+ pub const ATTR_LIST: &str = "SET UTF-8\nTRY esianrtolcdugmphbyfvkwzESIANRTOLCDUGMPHBYFVKWZ'\n\nREP 2\nREP f ph\nREP ph f\n\nPFX A Y 1\nPFX A 0 re .\n\nSFX B Y 2\nSFX B 0 ed [^y]\nSFX B y ied y";++ #[test]+ fn correctly_expands_test_files() {+ let words = parse_word_list(TEST_WORD_LIST).unwrap();+ let attributes = AttributeList::parse(ATTR_LIST).unwrap();++ dbg!(&attributes);++ let expanded = attributes.expand_marked_words(words).unwrap();+ let expanded: Vec<String> = expanded+ .into_iter()+ .map(|v| v.into_iter().collect())+ .collect();++ assert_eq!(+ expanded,+ vec!["hello", "tried", "try", "reworked", "rework", "worked", "work"]+ )+ }+}diff --git a/harper-core/src/spell/hunspell/word_list.rs b/harper-core/src/spell/hunspell/word_list.rsnew file mode 100644index 000000000..9bce1021d--- /dev/null+++ b/harper-core/src/spell/hunspell/word_list.rs@@ -0,0 +1,55 @@+use super::Error;++pub struct MarkedWord {+ pub letters: Vec<char>,+ pub attributes: Vec<char>,+}++pub struct Dictionary {+ pub(self) words: Vec<MarkedWord>,+}++/// Parse a hunspell word list+///+/// Returns [`None`] if the given string is invalid.+pub fn parse_word_list(source: &str) -> Result<Vec<MarkedWord>, Error> {+ let mut lines = source.lines();++ let approx_item_count = lines+ .next()+ .ok_or(Error::MalformedItemCount)?+ .parse()+ .map_err(|_| Error::MalformedItemCount)?;++ let mut words = Vec::with_capacity(approx_item_count);++ for line in lines {+ if let Some((word, attributes)) = line.split_once('/') {+ words.push(MarkedWord {+ letters: word.chars().collect(),+ attributes: attributes.chars().collect(),+ })+ } else {+ words.push(MarkedWord {+ letters: line.chars().collect(),+ attributes: Vec::new(),+ })+ }+ }++ Ok(words)+}++#[cfg(test)]+mod tests {+ use super::super::tests::TEST_WORD_LIST;+ use super::parse_word_list;++ #[test]+ fn can_parse_test_file() {+ let list = parse_word_list(TEST_WORD_LIST).unwrap();++ assert_eq!(list.last().unwrap().attributes.len(), 2);+ assert_eq!(list.len(), 3);+ }+}diff --git a/harper-core/src/spell/mod.rs b/harper-core/src/spell/mod.rsindex c74766baf..cf18aa1f7 100644--- a/harper-core/src/spell/mod.rs+++ b/harper-core/src/spell/mod.rs@@ -1,6 +1,7 @@pub use self::dictionary::Dictionary;mod dictionary;+mod hunspell;/// Suggest a correct spelling for a given misspelled word./// [misspelled_word] is assumed to be quite small (n < 100)