Skip to content

Offline, privacy-first grammar checker. Fast, open-source, Rust-powered

feat(fst-dict): mostly implement fuzzy-find

Grant Lemons4ff2db7c6e25
diff --git a/harper-core/src/spell/dictionary.rs b/harper-core/src/spell/dictionary.rs
index 71221413e..3fbcb0703 100644
--- a/harper-core/src/spell/dictionary.rs
+++ b/harper-core/src/spell/dictionary.rs
@@ -1,6 +1,6 @@
use blanket::blanket;
-use crate::{CharString, WordMetadata};
+use crate::WordMetadata;
#[cfg(not(feature = "concurrent"))]
#[blanket(derive(Rc))]
@@ -10,9 +10,19 @@ pub trait Dictionary {
/// Check if the dictionary contains a given word.
fn contains_word_str(&self, word: &str) -> bool;
/// Gets best fuzzy match from dictionary
- fn fuzzy_match(&self, word: &[char], max_distance: u8) -> Vec<(CharString, WordMetadata)>;
+ fn fuzzy_match(
+ &self,
+ word: &[char],
+ max_distance: u8,
+ max_results: usize,
+ ) -> Vec<(&[char], u8, WordMetadata)>;
/// Gets best fuzzy match from dictionary
- fn fuzzy_match_str(&self, word: &str, max_distance: u8) -> Vec<(CharString, WordMetadata)>;
+ fn fuzzy_match_str(
+ &self,
+ word: &str,
+ max_distance: u8,
+ max_results: usize,
+ ) -> Vec<(&[char], u8, WordMetadata)>;
/// Get the associated [`WordMetadata`] for a given word.
/// If the word isn't in the dictionary, the resulting metadata will be
/// empty.
@@ -31,9 +41,19 @@ pub trait Dictionary: Send + Sync {
/// Check if the dictionary contains a given word.
fn contains_word_str(&self, word: &str) -> bool;
/// Gets best fuzzy match from dictionary
- fn fuzzy_match(&self, word: &[char], max_distance: u8) -> Vec<(CharString, WordMetadata)>;
+ fn fuzzy_match(
+ &self,
+ word: &[char],
+ max_distance: u8,
+ max_results: usize,
+ ) -> Vec<(&[char], u8, WordMetadata)>;
/// Gets best fuzzy match from dictionary
- fn fuzzy_match_str(&self, word: &str, max_distance: u8) -> Vec<(CharString, WordMetadata)>;
+ fn fuzzy_match_str(
+ &self,
+ word: &str,
+ max_distance: u8,
+ max_results: usize,
+ ) -> Vec<(&[char], u8, WordMetadata)>;
/// Get the associated [`WordMetadata`] for a given word.
/// If the word isn't in the dictionary, the resulting metadata will be
/// empty.
diff --git a/harper-core/src/spell/fst_dictionary.rs b/harper-core/src/spell/fst_dictionary.rs
index 2e91132c8..046f95e85 100644
--- a/harper-core/src/spell/fst_dictionary.rs
+++ b/harper-core/src/spell/fst_dictionary.rs
@@ -5,7 +5,7 @@ use fst::{automaton::Levenshtein, IntoStreamer};
use memmap::Mmap;
use super::hunspell::{parse_default_attribute_list, parse_default_word_list};
-use crate::{CharString, Lrc, WordMetadata};
+use crate::{Lrc, WordMetadata};
use super::Dictionary;
@@ -29,7 +29,10 @@ fn uncached_inner_new() -> Lrc<FstDictionary> {
let mmap = unsafe { Mmap::map(&File::open("../../dictionary.fst").unwrap()).unwrap() };
let word_map = FstMap::new(mmap).unwrap();
- let mut words: Vec<CharString> = word_list.iter().map(|mw| mw.letters.clone()).collect();
+ let mut words: Vec<&[char]> = word_list
+ .iter()
+ .map(|mw| mw.letters.as_ref().into())
+ .collect();
words.sort();
words.dedup();
let metadata: Vec<WordMetadata> = todo!();
@@ -60,31 +63,27 @@ impl Dictionary for FstDictionary {
self.metadata[index]
}
- fn fuzzy_match(&self, word: &[char], max_distance: u8) -> Vec<(CharString, WordMetadata)> {
- let aut = Levenshtein::new(&word.iter().collect::<String>(), max_distance as u32).unwrap();
- let words = self
- .word_map
- .search(aut)
- .into_stream()
- .into_str_vec()
- .unwrap();
- words
- .into_iter()
- .map(|(word, i)| (word.chars().collect(), self.metadata[i as usize]))
- .collect()
+ fn fuzzy_match(
+ &self,
+ word: &[char],
+ max_distance: u8,
+ max_results: usize,
+ ) -> Vec<(&[char], u8, WordMetadata)> {
+ self.fuzzy_match_str(&word.iter().collect::<String>(), max_distance, max_results)
}
- fn fuzzy_match_str(&self, word: &str, max_distance: u8) -> Vec<(CharString, WordMetadata)> {
+ fn fuzzy_match_str(
+ &self,
+ word: &str,
+ max_distance: u8,
+ max_results: usize,
+ ) -> Vec<(&[char], u8, WordMetadata)> {
let aut = Levenshtein::new(word, max_distance as u32).unwrap();
- let words = self
- .word_map
- .search(aut)
- .into_stream()
- .into_str_vec()
- .unwrap();
+ let words: Vec<(Vec<u8>, u64)> = self.word_map.search(aut).into_stream().into_byte_vec();
words
.into_iter()
- .map(|(word, i)| (word.chars().collect(), self.metadata[i as usize]))
+ .take(max_results)
+ .map(|(word, i)| (word, i as u8, self.metadata[i as usize]))
.collect()
}
}
diff --git a/harper-core/src/spell/full_dictionary.rs b/harper-core/src/spell/full_dictionary.rs
index 4f6a4ea60..e6c0c1ca4 100644
--- a/harper-core/src/spell/full_dictionary.rs
+++ b/harper-core/src/spell/full_dictionary.rs
@@ -1,9 +1,10 @@
+use std::borrow::Cow;
+
use hashbrown::HashMap;
use smallvec::{SmallVec, ToSmallVec};
use super::dictionary::Dictionary;
use super::hunspell::{parse_default_attribute_list, parse_default_word_list};
-use super::seq_to_normalized;
use crate::{CharString, Lrc, WordMetadata};
/// A full, fat dictionary.
@@ -116,6 +117,26 @@ impl FullDictionary {
word_len_starts
}
+
+ fn words_iter(&self) -> impl Iterator<Item = &'_ [char]> {
+ self.words.iter().map(|v| v.as_slice())
+ }
+
+ /// Iterate over all the words in the dictionary of a given length
+ fn words_with_len_iter(&self, len: usize) -> Box<dyn Iterator<Item = &'_ [char]> + '_> {
+ if len == 0 || len >= self.word_len_starts.len() {
+ return Box::new(std::iter::empty());
+ }
+
+ let start = self.word_len_starts[len];
+ let end = if len + 1 == self.word_len_starts.len() {
+ self.words.len()
+ } else {
+ self.word_len_starts[len + 1]
+ };
+
+ Box::new(self.words[start..end].iter().map(|v| v.as_slice()))
+ }
}
impl Default for FullDictionary {
@@ -153,21 +174,174 @@ impl Dictionary for FullDictionary {
self.get_word_metadata(&chars)
}
- fn fuzzy_match(&self, word: &[char], max_distance: u8) -> Vec<(CharString, WordMetadata)> {
- todo!()
+ /// Suggest a correct spelling for a given misspelled word.
+ /// [`word`] is assumed to be quite small (n < 100).
+ /// [`max_distance`] relates to an optimization that allows the search
+ /// algorithm to prune large portions of the search.
+ fn fuzzy_match(
+ &self,
+ word: &[char],
+ max_distance: u8,
+ max_results: usize,
+ ) -> Vec<(&[char], u8, WordMetadata)> {
+ let misspelled_word = seq_to_normalized(word);
+ let misspelled_lower: Vec<char> = misspelled_word
+ .iter()
+ .flat_map(|v| v.to_lowercase())
+ .collect();
+
+ // 53 is the length of the longest word.
+ let mut buf_a = Vec::with_capacity(53);
+ let mut buf_b = Vec::with_capacity(53);
+
+ // The length of the shortest word to look at.
+ let shortest_word_len = if misspelled_word.len() < max_distance as usize {
+ 1
+ } else {
+ misspelled_word.len() - max_distance as usize
+ };
+
+ // Note how we look at the biggest words first.
+ let words_to_search = (shortest_word_len..misspelled_word.len() + max_distance as usize)
+ .rev()
+ .flat_map(|len| self.words_with_len_iter(len));
+
+ let pruned_words = words_to_search.filter_map(|word| {
+ let dist = edit_distance_min_alloc(&misspelled_word, word, &mut buf_a, &mut buf_b);
+ let dist_lower =
+ edit_distance_min_alloc(&misspelled_lower, word, &mut buf_a, &mut buf_b);
+
+ if dist.min(dist_lower) <= max_distance {
+ Some((word, dist))
+ } else {
+ None
+ }
+ });
+
+ // Locate the words with the lowest edit distance.
+ let mut found_dist: Vec<(&[char], u8)> = Vec::with_capacity(max_results);
+
+ for (word, dist) in pruned_words {
+ if found_dist.len() < max_results {
+ found_dist.push((word, dist));
+ } else if dist < found_dist[max_results - 1].1 {
+ found_dist[max_results - 1] = (word, dist);
+ }
+ found_dist.sort_by_key(|a| a.1);
+ }
+
+ // Create final, ordered list of suggestions.
+ found_dist
+ .into_iter()
+ .map(|v| (v.0, v.1, self.get_word_metadata(v.0)))
+ .collect()
+ }
+
+ fn fuzzy_match_str(
+ &self,
+ word: &str,
+ max_distance: u8,
+ max_results: usize,
+ ) -> Vec<(&[char], u8, WordMetadata)> {
+ let word: Vec<_> = word.chars().collect();
+ self.fuzzy_match(&word, max_distance, max_results)
+ }
+}
+
+// Computes the Levenstein edit distance between two patterns.
+// This is accomplished via a memory-optimized Wagner-Fischer algorithm
+//
+// This variant avoids allocation if you already have buffers.
+#[inline]
+pub fn edit_distance_min_alloc(
+ source: &[char],
+ target: &[char],
+ previous_row: &mut Vec<u8>,
+ current_row: &mut Vec<u8>,
+) -> u8 {
+ if cfg!(debug_assertions) {
+ assert!(source.len() <= 255 && target.len() <= 255);
+ }
+
+ let row_width = source.len();
+ let col_height = target.len();
+
+ previous_row.clear();
+ previous_row.extend(0u8..=row_width as u8);
+ // Alright if not zeroed, since we overwrite it anyway.
+ current_row.resize(row_width + 1, 0);
+
+ for j in 1..=col_height {
+ current_row[0] = j as u8;
+
+ for i in 1..=row_width {
+ let cost = if source[i - 1] == target[j - 1] { 0 } else { 1 };
+
+ current_row[i] = (previous_row[i] + 1)
+ .min(current_row[i - 1] + 1)
+ .min(previous_row[i - 1] + cost);
+ }
+
+ std::mem::swap(previous_row, current_row);
+ }
+
+ previous_row[row_width]
+}
+
+pub fn edit_distance(source: &[char], target: &[char]) -> u8 {
+ edit_distance_min_alloc(source, target, &mut Vec::new(), &mut Vec::new())
+}
+
+/// Convert a given character sequence to the standard character set
+/// the dictionary is in.
+pub fn seq_to_normalized(seq: &[char]) -> Cow<'_, [char]> {
+ if seq.iter().any(|c| char_to_normalized(*c) != *c) {
+ Cow::Owned(seq.iter().copied().map(char_to_normalized).collect())
+ } else {
+ Cow::Borrowed(seq)
}
+}
- fn fuzzy_match_str(&self, word: &str, max_distance: u8) -> Vec<(CharString, WordMetadata)> {
- todo!()
+pub fn char_to_normalized(c: char) -> char {
+ match c {
+ '’' => '\'',
+ _ => c,
}
}
#[cfg(test)]
mod tests {
+ use super::{edit_distance, seq_to_normalized};
use itertools::Itertools;
use crate::{Dictionary, FullDictionary};
+ fn assert_edit_dist(source: &str, target: &str, expected: u8) {
+ let source: Vec<_> = source.chars().collect();
+ let target: Vec<_> = target.chars().collect();
+
+ let dist = edit_distance(&source, &target);
+ assert_eq!(dist, expected)
+ }
+
+ #[test]
+ fn normalizes_weve() {
+ let word = vec!['w', 'e', '’', 'v', 'e'];
+ let norm = seq_to_normalized(&word);
+
+ assert_eq!(norm.clone(), vec!['w', 'e', '\'', 'v', 'e'])
+ }
+
+ #[test]
+ fn simple1() {
+ assert_edit_dist("kitten", "sitting", 3)
+ }
+
+ #[test]
+ fn simple2() {
+ assert_edit_dist("saturday", "sunday", 3)
+ }
+
#[test]
fn curated_contains_no_duplicates() {
let dict = FullDictionary::curated();
diff --git a/harper-core/src/spell/merged_dictionary.rs b/harper-core/src/spell/merged_dictionary.rs
index a3fb5a51a..7b1b3dc0d 100644
--- a/harper-core/src/spell/merged_dictionary.rs
+++ b/harper-core/src/spell/merged_dictionary.rs
@@ -1,6 +1,7 @@
use crate::Lrc;
use hashbrown::HashMap;
+use itertools::Itertools;
use super::dictionary::Dictionary;
use crate::{CharString, WordMetadata};
@@ -85,11 +86,31 @@ where
self.get_word_metadata(&chars)
}
- fn fuzzy_match(&self, word: &[char], max_distance: u8) -> Vec<(CharString, WordMetadata)> {
- todo!()
+ fn fuzzy_match(
+ &self,
+ word: &[char],
+ max_distance: u8,
+ max_results: usize,
+ ) -> Vec<(&[char], u8, WordMetadata)> {
+ self.children
+ .iter()
+ .flat_map(|d| d.fuzzy_match(word, max_distance, max_results))
+ .sorted_by_key(|r| r.1)
+ .take(max_results)
+ .collect()
}
- fn fuzzy_match_str(&self, word: &str, max_distance: u8) -> Vec<(CharString, WordMetadata)> {
- todo!()
+ fn fuzzy_match_str(
+ &self,
+ word: &str,
+ max_distance: u8,
+ max_results: usize,
+ ) -> Vec<(&[char], u8, WordMetadata)> {
+ self.children
+ .iter()
+ .flat_map(|d| d.fuzzy_match_str(word, max_distance, max_results))
+ .sorted_by_key(|r| r.1)
+ .take(max_results)
+ .collect()
}
}
diff --git a/harper-core/src/spell/mod.rs b/harper-core/src/spell/mod.rs
index 2868eb082..072f9aae7 100644
--- a/harper-core/src/spell/mod.rs
+++ b/harper-core/src/spell/mod.rs
@@ -2,6 +2,8 @@ use std::borrow::Cow;
use itertools::{Itertools, MinMaxResult};
+use crate::CharString;
+
pub use self::dictionary::Dictionary;
pub use self::full_dictionary::FullDictionary;
pub use self::merged_dictionary::MergedDictionary;
@@ -12,215 +14,42 @@ mod full_dictionary;
mod hunspell;
mod merged_dictionary;
-/// Suggest a correct spelling for a given misspelled word.
-/// [`misspelled_word`] is assumed to be quite small (n < 100).
-/// [`max_edit_dist`] relates to an optimization that allows the search
-/// algorithm to prune large portions of the search.
+/// Get the closest matches in the provided [`Dictionary`] and rank them
+/// Implementation is left up to the underlying dictionary.
pub fn suggest_correct_spelling<'a>(
misspelled_word: &[char],
result_limit: usize,
max_edit_dist: u8,
dictionary: &'a impl Dictionary,
) -> Vec<&'a [char]> {
- let misspelled_word = seq_to_normalized(misspelled_word);
-
- let misspelled_lower: Vec<char> = misspelled_word
- .iter()
- .flat_map(|v| v.to_lowercase())
+ let matches: Vec<&[char]> = dictionary
+ .fuzzy_match(misspelled_word, max_edit_dist, result_limit)
+ .into_iter()
+ .map(|r| r.0)
.collect();
-
- // 53 is the length of the longest word.
- let mut buf_a = Vec::with_capacity(53);
- let mut buf_b = Vec::with_capacity(53);
-
- // The length of the shortest word to look at.
- let shortest_word_len = if misspelled_word.len() < max_edit_dist as usize {
- 1
- } else {
- misspelled_word.len() - max_edit_dist as usize
- };
-
- // Note how we look at the biggest words first.
- let words_to_search = (shortest_word_len..misspelled_word.len() + max_edit_dist as usize)
- .rev()
- .flat_map(|len| dictionary.words_with_len_iter(len));
-
- let pruned_words = words_to_search.filter_map(|word| {
- let dist = edit_distance_min_alloc(&misspelled_word, word, &mut buf_a, &mut buf_b);
- let dist_lower = edit_distance_min_alloc(&misspelled_lower, word, &mut buf_a, &mut buf_b);
-
- if dist.min(dist_lower) <= max_edit_dist {
- Some((word, dist))
- } else {
- None
- }
- });
-
- // Locate the words with the lowest edit distance.
- let mut found_dist: Vec<(&[char], u8)> = Vec::with_capacity(result_limit);
-
- for (word, dist) in pruned_words {
- if found_dist.len() < result_limit {
- found_dist.push((word, dist));
- } else if dist < found_dist[result_limit - 1].1 {
- found_dist[result_limit - 1] = (word, dist);
- }
- found_dist.sort_by_key(|a| a.1);
- }
-
- // Create final, ordered list of suggestions.
- let mut found = Vec::with_capacity(found_dist.len());
-
- // Often the longest and the shortest words are the most helpful, so lets push
- // them first.
- let minmax = found_dist.iter().position_minmax_by_key(|a| a.0.len());
- if let MinMaxResult::MinMax(a, b) = minmax {
- if a == b {
- found.push(found_dist[a].0);
- } else {
- found.push(found_dist[a].0);
- found.push(found_dist[b].0);
- }
-
- // Push the rest
- found.extend(
- found_dist
- .into_iter()
- .enumerate()
- .filter(|(i, _)| *i != a && *i != b)
- .map(|v| v.1 .0),
- );
- } else {
- // Push the rest
- found.extend(found_dist.into_iter().map(|v| v.0));
- }
-
- // Swap the lowest edit distance word with the shortest.
- if found.len() >= 3 {
- found.swap(0, 2);
- }
-
- // Let common words bubble up.
- found.sort_by_key(|v| {
- if dictionary.get_word_metadata(v).common {
- 0
- } else {
- 1
- }
- });
-
- found
+ matches
}
/// Convenience function over [`suggest_correct_spelling`] that does conversions
/// for you.
-pub fn suggest_correct_spelling_str(
- misspelled_word: impl AsRef<str>,
+pub(self) fn suggest_correct_spelling_str(
+ misspelled_word: impl Into<String>,
result_limit: usize,
max_edit_dist: u8,
dictionary: &FullDictionary,
) -> Vec<String> {
- let chars: Vec<char> = misspelled_word.as_ref().chars().collect();
-
+ let chars: CharString = misspelled_word.into().chars().collect();
suggest_correct_spelling(&chars, result_limit, max_edit_dist, dictionary)
.into_iter()
- .map(|word| word.iter().collect())
+ .map(|a| a.iter().collect::<String>())
.collect()
}
-// Computes the Levenstein edit distance between two patterns.
-// This is accomplished via a memory-optimized Wagner-Fischer algorithm
-//
-// This variant avoids allocation if you already have buffers.
-#[inline]
-fn edit_distance_min_alloc(
- source: &[char],
- target: &[char],
- previous_row: &mut Vec<u8>,
- current_row: &mut Vec<u8>,
-) -> u8 {
- if cfg!(debug_assertions) {
- assert!(source.len() <= 255 && target.len() <= 255);
- }
-
- let row_width = source.len();
- let col_height = target.len();
-
- previous_row.clear();
- previous_row.extend(0u8..=row_width as u8);
- // Alright if not zeroed, since we overwrite it anyway.
- current_row.resize(row_width + 1, 0);
-
- for j in 1..=col_height {
- current_row[0] = j as u8;
-
- for i in 1..=row_width {
- let cost = if source[i - 1] == target[j - 1] { 0 } else { 1 };
-
- current_row[i] = (previous_row[i] + 1)
- .min(current_row[i - 1] + 1)
- .min(previous_row[i - 1] + cost);
- }
-
- std::mem::swap(previous_row, current_row);
- }
-
- previous_row[row_width]
-}
-
-fn edit_distance(source: &[char], target: &[char]) -> u8 {
- edit_distance_min_alloc(source, target, &mut Vec::new(), &mut Vec::new())
-}
-
-/// Convert a given character sequence to the standard character set
-/// the dictionary is in.
-fn seq_to_normalized(seq: &[char]) -> Cow<'_, [char]> {
- if seq.iter().any(|c| char_to_normalized(*c) != *c) {
- Cow::Owned(seq.iter().copied().map(char_to_normalized).collect())
- } else {
- Cow::Borrowed(seq)
- }
-}
-
-fn char_to_normalized(c: char) -> char {
- match c {
- '’' => '\'',
- _ => c,
- }
-}
-
#[cfg(test)]
mod tests {
use itertools::Itertools;
- use super::{edit_distance, seq_to_normalized, suggest_correct_spelling_str};
- use crate::FullDictionary;
-
- fn assert_edit_dist(source: &str, target: &str, expected: u8) {
- let source: Vec<_> = source.chars().collect();
- let target: Vec<_> = target.chars().collect();
-
- let dist = edit_distance(&source, &target);
- assert_eq!(dist, expected)
- }
-
- #[test]
- fn normalizes_weve() {
- let word = vec!['w', 'e', '’', 'v', 'e'];
- let norm = seq_to_normalized(&word);
-
- assert_eq!(norm.clone(), vec!['w', 'e', '\'', 'v', 'e'])
- }
-
- #[test]
- fn simple1() {
- assert_edit_dist("kitten", "sitting", 3)
- }
-
- #[test]
- fn simple2() {
- assert_edit_dist("saturday", "sunday", 3)
- }
+ use crate::{spell::suggest_correct_spelling_str, FullDictionary};
#[test]
fn produces_no_duplicates() {