Offline, privacy-first grammar checker. Fast, open-source, Rust-powered
feat(fst-dict): mostly implement fuzzy-find
Grant Lemons4ff2db7c6e25
diff --git a/harper-core/src/spell/dictionary.rs b/harper-core/src/spell/dictionary.rsindex 71221413e..3fbcb0703 100644--- a/harper-core/src/spell/dictionary.rs+++ b/harper-core/src/spell/dictionary.rs@@ -1,6 +1,6 @@use blanket::blanket;-use crate::{CharString, WordMetadata};+use crate::WordMetadata;#[cfg(not(feature = "concurrent"))]#[blanket(derive(Rc))]@@ -10,9 +10,19 @@ pub trait Dictionary {/// Check if the dictionary contains a given word.fn contains_word_str(&self, word: &str) -> bool;/// Gets best fuzzy match from dictionary- fn fuzzy_match(&self, word: &[char], max_distance: u8) -> Vec<(CharString, WordMetadata)>;+ fn fuzzy_match(+ &self,+ word: &[char],+ max_distance: u8,+ max_results: usize,+ ) -> Vec<(&[char], u8, WordMetadata)>;/// Gets best fuzzy match from dictionary- fn fuzzy_match_str(&self, word: &str, max_distance: u8) -> Vec<(CharString, WordMetadata)>;+ fn fuzzy_match_str(+ &self,+ word: &str,+ max_distance: u8,+ max_results: usize,+ ) -> Vec<(&[char], u8, WordMetadata)>;/// Get the associated [`WordMetadata`] for a given word./// If the word isn't in the dictionary, the resulting metadata will be/// empty.@@ -31,9 +41,19 @@ pub trait Dictionary: Send + Sync {/// Check if the dictionary contains a given word.fn contains_word_str(&self, word: &str) -> bool;/// Gets best fuzzy match from dictionary- fn fuzzy_match(&self, word: &[char], max_distance: u8) -> Vec<(CharString, WordMetadata)>;+ fn fuzzy_match(+ &self,+ word: &[char],+ max_distance: u8,+ max_results: usize,+ ) -> Vec<(&[char], u8, WordMetadata)>;/// Gets best fuzzy match from dictionary- fn fuzzy_match_str(&self, word: &str, max_distance: u8) -> Vec<(CharString, WordMetadata)>;+ fn fuzzy_match_str(+ &self,+ word: &str,+ max_distance: u8,+ max_results: usize,+ ) -> Vec<(&[char], u8, WordMetadata)>;/// Get the associated [`WordMetadata`] for a given word./// If the word isn't in the dictionary, the resulting metadata will be/// empty.diff --git a/harper-core/src/spell/fst_dictionary.rs b/harper-core/src/spell/fst_dictionary.rsindex 2e91132c8..046f95e85 100644--- a/harper-core/src/spell/fst_dictionary.rs+++ b/harper-core/src/spell/fst_dictionary.rs@@ -5,7 +5,7 @@ use fst::{automaton::Levenshtein, IntoStreamer};use memmap::Mmap;use super::hunspell::{parse_default_attribute_list, parse_default_word_list};-use crate::{CharString, Lrc, WordMetadata};+use crate::{Lrc, WordMetadata};use super::Dictionary;@@ -29,7 +29,10 @@ fn uncached_inner_new() -> Lrc<FstDictionary> {let mmap = unsafe { Mmap::map(&File::open("../../dictionary.fst").unwrap()).unwrap() };let word_map = FstMap::new(mmap).unwrap();- let mut words: Vec<CharString> = word_list.iter().map(|mw| mw.letters.clone()).collect();+ let mut words: Vec<&[char]> = word_list+ .iter()+ .map(|mw| mw.letters.as_ref().into())+ .collect();words.sort();words.dedup();let metadata: Vec<WordMetadata> = todo!();@@ -60,31 +63,27 @@ impl Dictionary for FstDictionary {self.metadata[index]}- fn fuzzy_match(&self, word: &[char], max_distance: u8) -> Vec<(CharString, WordMetadata)> {- let aut = Levenshtein::new(&word.iter().collect::<String>(), max_distance as u32).unwrap();- let words = self- .word_map- .search(aut)- .into_stream()- .into_str_vec()- .unwrap();- words- .into_iter()- .map(|(word, i)| (word.chars().collect(), self.metadata[i as usize]))- .collect()+ fn fuzzy_match(+ &self,+ word: &[char],+ max_distance: u8,+ max_results: usize,+ ) -> Vec<(&[char], u8, WordMetadata)> {+ self.fuzzy_match_str(&word.iter().collect::<String>(), max_distance, max_results)}- fn fuzzy_match_str(&self, word: &str, max_distance: u8) -> Vec<(CharString, WordMetadata)> {+ fn fuzzy_match_str(+ &self,+ word: &str,+ max_distance: u8,+ max_results: usize,+ ) -> Vec<(&[char], u8, WordMetadata)> {let aut = Levenshtein::new(word, max_distance as u32).unwrap();- let words = self- .word_map- .search(aut)- .into_stream()- .into_str_vec()- .unwrap();+ let words: Vec<(Vec<u8>, u64)> = self.word_map.search(aut).into_stream().into_byte_vec();words.into_iter()- .map(|(word, i)| (word.chars().collect(), self.metadata[i as usize]))+ .take(max_results)+ .map(|(word, i)| (word, i as u8, self.metadata[i as usize])).collect()}}diff --git a/harper-core/src/spell/full_dictionary.rs b/harper-core/src/spell/full_dictionary.rsindex 4f6a4ea60..e6c0c1ca4 100644--- a/harper-core/src/spell/full_dictionary.rs+++ b/harper-core/src/spell/full_dictionary.rs@@ -1,9 +1,10 @@+use std::borrow::Cow;+use hashbrown::HashMap;use smallvec::{SmallVec, ToSmallVec};use super::dictionary::Dictionary;use super::hunspell::{parse_default_attribute_list, parse_default_word_list};-use super::seq_to_normalized;use crate::{CharString, Lrc, WordMetadata};/// A full, fat dictionary.@@ -116,6 +117,26 @@ impl FullDictionary {word_len_starts}++ fn words_iter(&self) -> impl Iterator<Item = &'_ [char]> {+ self.words.iter().map(|v| v.as_slice())+ }++ /// Iterate over all the words in the dictionary of a given length+ fn words_with_len_iter(&self, len: usize) -> Box<dyn Iterator<Item = &'_ [char]> + '_> {+ if len == 0 || len >= self.word_len_starts.len() {+ return Box::new(std::iter::empty());+ }++ let start = self.word_len_starts[len];+ let end = if len + 1 == self.word_len_starts.len() {+ self.words.len()+ } else {+ self.word_len_starts[len + 1]+ };++ Box::new(self.words[start..end].iter().map(|v| v.as_slice()))+ }}impl Default for FullDictionary {@@ -153,21 +174,174 @@ impl Dictionary for FullDictionary {self.get_word_metadata(&chars)}- fn fuzzy_match(&self, word: &[char], max_distance: u8) -> Vec<(CharString, WordMetadata)> {- todo!()+ /// Suggest a correct spelling for a given misspelled word.+ /// [`word`] is assumed to be quite small (n < 100).+ /// [`max_distance`] relates to an optimization that allows the search+ /// algorithm to prune large portions of the search.+ fn fuzzy_match(+ &self,+ word: &[char],+ max_distance: u8,+ max_results: usize,+ ) -> Vec<(&[char], u8, WordMetadata)> {+ let misspelled_word = seq_to_normalized(word);+ let misspelled_lower: Vec<char> = misspelled_word+ .iter()+ .flat_map(|v| v.to_lowercase())+ .collect();++ // 53 is the length of the longest word.+ let mut buf_a = Vec::with_capacity(53);+ let mut buf_b = Vec::with_capacity(53);++ // The length of the shortest word to look at.+ let shortest_word_len = if misspelled_word.len() < max_distance as usize {+ 1+ } else {+ misspelled_word.len() - max_distance as usize+ };++ // Note how we look at the biggest words first.+ let words_to_search = (shortest_word_len..misspelled_word.len() + max_distance as usize)+ .rev()+ .flat_map(|len| self.words_with_len_iter(len));++ let pruned_words = words_to_search.filter_map(|word| {+ let dist = edit_distance_min_alloc(&misspelled_word, word, &mut buf_a, &mut buf_b);+ let dist_lower =+ edit_distance_min_alloc(&misspelled_lower, word, &mut buf_a, &mut buf_b);++ if dist.min(dist_lower) <= max_distance {+ Some((word, dist))+ } else {+ None+ }+ });++ // Locate the words with the lowest edit distance.+ let mut found_dist: Vec<(&[char], u8)> = Vec::with_capacity(max_results);++ for (word, dist) in pruned_words {+ if found_dist.len() < max_results {+ found_dist.push((word, dist));+ } else if dist < found_dist[max_results - 1].1 {+ found_dist[max_results - 1] = (word, dist);+ }+ found_dist.sort_by_key(|a| a.1);+ }++ // Create final, ordered list of suggestions.+ found_dist+ .into_iter()+ .map(|v| (v.0, v.1, self.get_word_metadata(v.0)))+ .collect()+ }++ fn fuzzy_match_str(+ &self,+ word: &str,+ max_distance: u8,+ max_results: usize,+ ) -> Vec<(&[char], u8, WordMetadata)> {+ let word: Vec<_> = word.chars().collect();+ self.fuzzy_match(&word, max_distance, max_results)+ }+}++// Computes the Levenstein edit distance between two patterns.+// This is accomplished via a memory-optimized Wagner-Fischer algorithm+//+// This variant avoids allocation if you already have buffers.+#[inline]+pub fn edit_distance_min_alloc(+ source: &[char],+ target: &[char],+ previous_row: &mut Vec<u8>,+ current_row: &mut Vec<u8>,+) -> u8 {+ if cfg!(debug_assertions) {+ assert!(source.len() <= 255 && target.len() <= 255);+ }++ let row_width = source.len();+ let col_height = target.len();++ previous_row.clear();+ previous_row.extend(0u8..=row_width as u8);+ // Alright if not zeroed, since we overwrite it anyway.+ current_row.resize(row_width + 1, 0);++ for j in 1..=col_height {+ current_row[0] = j as u8;++ for i in 1..=row_width {+ let cost = if source[i - 1] == target[j - 1] { 0 } else { 1 };++ current_row[i] = (previous_row[i] + 1)+ .min(current_row[i - 1] + 1)+ .min(previous_row[i - 1] + cost);+ }++ std::mem::swap(previous_row, current_row);+ }++ previous_row[row_width]+}++pub fn edit_distance(source: &[char], target: &[char]) -> u8 {+ edit_distance_min_alloc(source, target, &mut Vec::new(), &mut Vec::new())+}++/// Convert a given character sequence to the standard character set+/// the dictionary is in.+pub fn seq_to_normalized(seq: &[char]) -> Cow<'_, [char]> {+ if seq.iter().any(|c| char_to_normalized(*c) != *c) {+ Cow::Owned(seq.iter().copied().map(char_to_normalized).collect())+ } else {+ Cow::Borrowed(seq)}+}- fn fuzzy_match_str(&self, word: &str, max_distance: u8) -> Vec<(CharString, WordMetadata)> {- todo!()+pub fn char_to_normalized(c: char) -> char {+ match c {+ '’' => '\'',+ _ => c,}}#[cfg(test)]mod tests {+ use super::{edit_distance, seq_to_normalized};use itertools::Itertools;use crate::{Dictionary, FullDictionary};+ fn assert_edit_dist(source: &str, target: &str, expected: u8) {+ let source: Vec<_> = source.chars().collect();+ let target: Vec<_> = target.chars().collect();++ let dist = edit_distance(&source, &target);+ assert_eq!(dist, expected)+ }++ #[test]+ fn normalizes_weve() {+ let word = vec!['w', 'e', '’', 'v', 'e'];+ let norm = seq_to_normalized(&word);++ assert_eq!(norm.clone(), vec!['w', 'e', '\'', 'v', 'e'])+ }++ #[test]+ fn simple1() {+ assert_edit_dist("kitten", "sitting", 3)+ }++ #[test]+ fn simple2() {+ assert_edit_dist("saturday", "sunday", 3)+ }+#[test]fn curated_contains_no_duplicates() {let dict = FullDictionary::curated();diff --git a/harper-core/src/spell/merged_dictionary.rs b/harper-core/src/spell/merged_dictionary.rsindex a3fb5a51a..7b1b3dc0d 100644--- a/harper-core/src/spell/merged_dictionary.rs+++ b/harper-core/src/spell/merged_dictionary.rs@@ -1,6 +1,7 @@use crate::Lrc;use hashbrown::HashMap;+use itertools::Itertools;use super::dictionary::Dictionary;use crate::{CharString, WordMetadata};@@ -85,11 +86,31 @@ whereself.get_word_metadata(&chars)}- fn fuzzy_match(&self, word: &[char], max_distance: u8) -> Vec<(CharString, WordMetadata)> {- todo!()+ fn fuzzy_match(+ &self,+ word: &[char],+ max_distance: u8,+ max_results: usize,+ ) -> Vec<(&[char], u8, WordMetadata)> {+ self.children+ .iter()+ .flat_map(|d| d.fuzzy_match(word, max_distance, max_results))+ .sorted_by_key(|r| r.1)+ .take(max_results)+ .collect()}- fn fuzzy_match_str(&self, word: &str, max_distance: u8) -> Vec<(CharString, WordMetadata)> {- todo!()+ fn fuzzy_match_str(+ &self,+ word: &str,+ max_distance: u8,+ max_results: usize,+ ) -> Vec<(&[char], u8, WordMetadata)> {+ self.children+ .iter()+ .flat_map(|d| d.fuzzy_match_str(word, max_distance, max_results))+ .sorted_by_key(|r| r.1)+ .take(max_results)+ .collect()}}diff --git a/harper-core/src/spell/mod.rs b/harper-core/src/spell/mod.rsindex 2868eb082..072f9aae7 100644--- a/harper-core/src/spell/mod.rs+++ b/harper-core/src/spell/mod.rs@@ -2,6 +2,8 @@ use std::borrow::Cow;use itertools::{Itertools, MinMaxResult};+use crate::CharString;+pub use self::dictionary::Dictionary;pub use self::full_dictionary::FullDictionary;pub use self::merged_dictionary::MergedDictionary;@@ -12,215 +14,42 @@ mod full_dictionary;mod hunspell;mod merged_dictionary;-/// Suggest a correct spelling for a given misspelled word.-/// [`misspelled_word`] is assumed to be quite small (n < 100).-/// [`max_edit_dist`] relates to an optimization that allows the search-/// algorithm to prune large portions of the search.+/// Get the closest matches in the provided [`Dictionary`] and rank them+/// Implementation is left up to the underlying dictionary.pub fn suggest_correct_spelling<'a>(misspelled_word: &[char],result_limit: usize,max_edit_dist: u8,dictionary: &'a impl Dictionary,) -> Vec<&'a [char]> {- let misspelled_word = seq_to_normalized(misspelled_word);-- let misspelled_lower: Vec<char> = misspelled_word- .iter()- .flat_map(|v| v.to_lowercase())+ let matches: Vec<&[char]> = dictionary+ .fuzzy_match(misspelled_word, max_edit_dist, result_limit)+ .into_iter()+ .map(|r| r.0).collect();-- // 53 is the length of the longest word.- let mut buf_a = Vec::with_capacity(53);- let mut buf_b = Vec::with_capacity(53);-- // The length of the shortest word to look at.- let shortest_word_len = if misspelled_word.len() < max_edit_dist as usize {- 1- } else {- misspelled_word.len() - max_edit_dist as usize- };-- // Note how we look at the biggest words first.- let words_to_search = (shortest_word_len..misspelled_word.len() + max_edit_dist as usize)- .rev()- .flat_map(|len| dictionary.words_with_len_iter(len));-- let pruned_words = words_to_search.filter_map(|word| {- let dist = edit_distance_min_alloc(&misspelled_word, word, &mut buf_a, &mut buf_b);- let dist_lower = edit_distance_min_alloc(&misspelled_lower, word, &mut buf_a, &mut buf_b);-- if dist.min(dist_lower) <= max_edit_dist {- Some((word, dist))- } else {- None- }- });-- // Locate the words with the lowest edit distance.- let mut found_dist: Vec<(&[char], u8)> = Vec::with_capacity(result_limit);-- for (word, dist) in pruned_words {- if found_dist.len() < result_limit {- found_dist.push((word, dist));- } else if dist < found_dist[result_limit - 1].1 {- found_dist[result_limit - 1] = (word, dist);- }- found_dist.sort_by_key(|a| a.1);- }-- // Create final, ordered list of suggestions.- let mut found = Vec::with_capacity(found_dist.len());-- // Often the longest and the shortest words are the most helpful, so lets push- // them first.- let minmax = found_dist.iter().position_minmax_by_key(|a| a.0.len());- if let MinMaxResult::MinMax(a, b) = minmax {- if a == b {- found.push(found_dist[a].0);- } else {- found.push(found_dist[a].0);- found.push(found_dist[b].0);- }-- // Push the rest- found.extend(- found_dist- .into_iter()- .enumerate()- .filter(|(i, _)| *i != a && *i != b)- .map(|v| v.1 .0),- );- } else {- // Push the rest- found.extend(found_dist.into_iter().map(|v| v.0));- }-- // Swap the lowest edit distance word with the shortest.- if found.len() >= 3 {- found.swap(0, 2);- }-- // Let common words bubble up.- found.sort_by_key(|v| {- if dictionary.get_word_metadata(v).common {- 0- } else {- 1- }- });-- found+ matches}/// Convenience function over [`suggest_correct_spelling`] that does conversions/// for you.-pub fn suggest_correct_spelling_str(- misspelled_word: impl AsRef<str>,+pub(self) fn suggest_correct_spelling_str(+ misspelled_word: impl Into<String>,result_limit: usize,max_edit_dist: u8,dictionary: &FullDictionary,) -> Vec<String> {- let chars: Vec<char> = misspelled_word.as_ref().chars().collect();-+ let chars: CharString = misspelled_word.into().chars().collect();suggest_correct_spelling(&chars, result_limit, max_edit_dist, dictionary).into_iter()- .map(|word| word.iter().collect())+ .map(|a| a.iter().collect::<String>()).collect()}-// Computes the Levenstein edit distance between two patterns.-// This is accomplished via a memory-optimized Wagner-Fischer algorithm-//-// This variant avoids allocation if you already have buffers.-#[inline]-fn edit_distance_min_alloc(- source: &[char],- target: &[char],- previous_row: &mut Vec<u8>,- current_row: &mut Vec<u8>,-) -> u8 {- if cfg!(debug_assertions) {- assert!(source.len() <= 255 && target.len() <= 255);- }-- let row_width = source.len();- let col_height = target.len();-- previous_row.clear();- previous_row.extend(0u8..=row_width as u8);- // Alright if not zeroed, since we overwrite it anyway.- current_row.resize(row_width + 1, 0);-- for j in 1..=col_height {- current_row[0] = j as u8;-- for i in 1..=row_width {- let cost = if source[i - 1] == target[j - 1] { 0 } else { 1 };-- current_row[i] = (previous_row[i] + 1)- .min(current_row[i - 1] + 1)- .min(previous_row[i - 1] + cost);- }-- std::mem::swap(previous_row, current_row);- }-- previous_row[row_width]-}--fn edit_distance(source: &[char], target: &[char]) -> u8 {- edit_distance_min_alloc(source, target, &mut Vec::new(), &mut Vec::new())-}--/// Convert a given character sequence to the standard character set-/// the dictionary is in.-fn seq_to_normalized(seq: &[char]) -> Cow<'_, [char]> {- if seq.iter().any(|c| char_to_normalized(*c) != *c) {- Cow::Owned(seq.iter().copied().map(char_to_normalized).collect())- } else {- Cow::Borrowed(seq)- }-}--fn char_to_normalized(c: char) -> char {- match c {- '’' => '\'',- _ => c,- }-}-#[cfg(test)]mod tests {use itertools::Itertools;- use super::{edit_distance, seq_to_normalized, suggest_correct_spelling_str};- use crate::FullDictionary;-- fn assert_edit_dist(source: &str, target: &str, expected: u8) {- let source: Vec<_> = source.chars().collect();- let target: Vec<_> = target.chars().collect();-- let dist = edit_distance(&source, &target);- assert_eq!(dist, expected)- }-- #[test]- fn normalizes_weve() {- let word = vec!['w', 'e', '’', 'v', 'e'];- let norm = seq_to_normalized(&word);-- assert_eq!(norm.clone(), vec!['w', 'e', '\'', 'v', 'e'])- }-- #[test]- fn simple1() {- assert_edit_dist("kitten", "sitting", 3)- }-- #[test]- fn simple2() {- assert_edit_dist("saturday", "sunday", 3)- }+ use crate::{spell::suggest_correct_spelling_str, FullDictionary};#[test]fn produces_no_duplicates() {