Offline, privacy-first grammar checker. Fast, open-source, Rust-powered
feat(fst-dict): write some tests and change fst fuzzy search to order by edit distance after search
Grant Lemons5f42ba33c258
diff --git a/harper-core/src/char_string.rs b/harper-core/src/char_string.rsindex 3a30889c6..899f5bff8 100644--- a/harper-core/src/char_string.rs+++ b/harper-core/src/char_string.rs@@ -6,6 +6,7 @@ pub type CharString = SmallVec<[char; 12]>;pub trait CharStringExt {fn to_lower(&self) -> CharString;+ fn to_string(&self) -> String;}impl CharStringExt for [char] {@@ -16,4 +17,7 @@ impl CharStringExt for [char] {out}+ fn to_string(&self) -> String {+ self.iter().collect()+ }}diff --git a/harper-core/src/spell/fst_dictionary.rs b/harper-core/src/spell/fst_dictionary.rsindex 5a69d1116..68f2d0b14 100644--- a/harper-core/src/spell/fst_dictionary.rs+++ b/harper-core/src/spell/fst_dictionary.rs@@ -1,8 +1,9 @@-use super::{seq_to_normalized, FullDictionary};+use super::{edit_distance_min_alloc, seq_to_normalized, FullDictionary};use fst::{automaton::Levenshtein, IntoStreamer};use fst::{Map as FstMap, Streamer};+use itertools::Itertools;-use crate::{Lrc, WordMetadata};+use crate::{CharStringExt, Lrc, WordMetadata};use super::Dictionary;@@ -76,49 +77,74 @@ impl Dictionary for FstDictionary {max_distance: u8,max_results: usize,) -> Vec<(&[char], u8, WordMetadata)> {+ // Various transformations of the inputlet chars: Vec<_> = word.chars().collect();- let misspelled_normalized = seq_to_normalized(&chars);- let misspelled_word: String = misspelled_normalized.iter().collect();- let misspelled_lower: String = misspelled_normalized- .iter()- .flat_map(|v| v.to_lowercase())- .collect();-- let aut = Levenshtein::new(&misspelled_word, max_distance as u32).unwrap();- let aut_lower = Levenshtein::new(&misspelled_lower, max_distance as u32).unwrap();- let mut word_lower_indexes_stream = self.word_map.search(aut_lower).into_stream();- let mut word_indexes_stream = self.word_map.search(aut).into_stream();-- let mut word_indexes = Vec::with_capacity(max_results);- let mut i = 0;- while i < max_results {+ let misspelled_word_charslice = seq_to_normalized(&chars);+ let misspelled_lower_charslice = misspelled_word_charslice.to_lower();+ let misspelled_word_string = misspelled_word_charslice.to_string();+ let misspelled_lower_string = misspelled_lower_charslice.to_string();++ // Actual FST search+ let automaton = Levenshtein::new(&misspelled_word_string, max_distance as u32).unwrap();+ let automaton_lower =+ Levenshtein::new(&misspelled_lower_string, max_distance as u32).unwrap();+ let mut word_lower_indexes_stream = self.word_map.search(automaton_lower).into_stream();+ let mut word_indexes_stream = self.word_map.search(automaton).into_stream();++ // Consume at most max_results values from each stream+ // The search itself happens as you consume from the stream, so consuming a smaller number+ // is more efficient+ // HashSet used to dedup results+ let mut words = hashbrown::HashSet::with_capacity(max_results * 2);+ for _ in 0..max_results {let mut flag = false;- if let Some(v) = word_lower_indexes_stream.next() {- word_indexes.push(v.1);+ if let Some((_, index)) = word_lower_indexes_stream.next() {+ words.insert(index);flag = true;}- if let Some(v) = word_indexes_stream.next() {- word_indexes.push(v.1);+ if let Some((_, index)) = word_indexes_stream.next() {+ words.insert(index);flag = true;}if !flag {break;}- i += 1;}- // Dedup but preserve order of word indexes- let mut seen = hashbrown::HashSet::new();- word_indexes.retain(|item| seen.insert(*item));- word_indexes+ // Pre-allocated vectors for edit-distance calculation+ // 53 is the length of the longest word.+ let mut buf_a = Vec::with_capacity(53);+ let mut buf_b = Vec::with_capacity(53);++ words.into_iter()+ .map(|index| (self.full_dict.get_word(index as usize), index))+ // Sort by edit distance+ .map(|(word, index)| {+ let dist = edit_distance_min_alloc(+ &misspelled_word_charslice,+ word,+ &mut buf_a,+ &mut buf_b,+ );+ buf_a.clear();+ buf_b.clear();+ let dist_lower = edit_distance_min_alloc(+ &misspelled_lower_charslice,+ word,+ &mut buf_a,+ &mut buf_b,+ );++ (std::cmp::min(dist, dist_lower), word, index)+ })+ .sorted_unstable_by_key(|a| a.0).take(max_results)- .map(|i| (self.full_dict.get_word(i as usize), i))- .map(|(word, i)| {+ .map(|(dist, word, index)| {(word.as_slice(),- i as u8,- self.full_dict.get_metadata(i as usize).to_owned(),+ dist,+ self.full_dict.get_metadata(index as usize).to_owned(),)}).collect()@@ -135,7 +161,10 @@ impl Dictionary for FstDictionary {#[cfg(test)]mod tests {- use crate::{spell::seq_to_normalized, Dictionary};+ use fst::IntoStreamer;+ use itertools::Itertools;++ use crate::{spell::seq_to_normalized, CharStringExt, Dictionary};use super::FstDictionary;@@ -145,11 +174,8 @@ mod tests {for word in dict.words_iter() {let misspelled_normalized = seq_to_normalized(word);- let misspelled_word: String = misspelled_normalized.iter().collect();- let misspelled_lower: String = misspelled_normalized- .iter()- .flat_map(|v| v.to_lowercase())- .collect();+ let misspelled_word: String = misspelled_normalized.to_string();+ let misspelled_lower: String = misspelled_normalized.to_lower().to_string();assert!(!misspelled_word.is_empty());assert!(@@ -165,11 +191,8 @@ mod tests {let word: Vec<_> = "hello".chars().collect();let misspelled_normalized = seq_to_normalized(&word);- let misspelled_word: String = misspelled_normalized.iter().collect();- let misspelled_lower: String = misspelled_normalized- .iter()- .flat_map(|v| v.to_lowercase())- .collect();+ let misspelled_word: String = misspelled_normalized.to_string();+ let misspelled_lower: String = misspelled_normalized.to_lower().to_string();assert!(dict.contains_word(&misspelled_normalized));assert!(@@ -177,4 +200,49 @@ mod tests {|| dict.word_map.contains_key(misspelled_word));}++ #[test]+ fn fst_search_hello() {+ let dict = FstDictionary::curated();++ let word: Vec<_> = "hvllo".chars().collect();+ let misspelled_normalized = seq_to_normalized(&word);+ let misspelled_word: String = misspelled_normalized.to_string();+ let misspelled_lower: String = misspelled_normalized.to_lower().to_string();++ let aut = fst::automaton::Levenshtein::new(&misspelled_word, 2).unwrap();+ let aut_lower = fst::automaton::Levenshtein::new(&misspelled_lower, 2).unwrap();+ let word_indexes_stream = dict+ .word_map+ .search(aut)+ .into_stream()+ .into_str_keys()+ .unwrap();+ let word_lower_indexes_stream = dict+ .word_map+ .search(aut_lower)+ .into_stream()+ .into_str_keys()+ .unwrap();++ dbg!(&word_indexes_stream);+ assert!(+ word_indexes_stream.contains(&"hello".to_string())+ || word_lower_indexes_stream.contains(&"hello".to_string())+ );+ }++ #[test]+ fn fuzzy_hello() {+ let dict = FstDictionary::curated();++ let matches: Vec<_> = dict+ .fuzzy_match_str("hvllo", 3, 100)+ .iter()+ .map(|cs| cs.0.iter().collect::<String>())+ .collect();+ dbg!(&matches);++ assert!(matches.iter().take(10).contains(&"hello".to_string()))+ }}diff --git a/harper-core/src/spell/full_dictionary.rs b/harper-core/src/spell/full_dictionary.rsindex b6af4063b..657a92e21 100644--- a/harper-core/src/spell/full_dictionary.rs+++ b/harper-core/src/spell/full_dictionary.rs@@ -1,9 +1,9 @@-use super::seq_to_normalized;+use super::{edit_distance_min_alloc, seq_to_normalized};use hashbrown::HashMap;use smallvec::{SmallVec, ToSmallVec};use super::dictionary::Dictionary;-use crate::{CharString, Lrc, WordMetadata};+use crate::{CharString, CharStringExt, Lrc, WordMetadata};use harper_dictionary_parsing::{parse_default_attribute_list, parse_default_word_list};/// A full, fat dictionary.@@ -136,7 +136,7 @@ impl Default for FullDictionary {impl Dictionary for FullDictionary {fn get_word_metadata(&self, word: &[char]) -> WordMetadata {let normalized = seq_to_normalized(word);- let lowercase: CharString = normalized.iter().flat_map(|c| c.to_lowercase()).collect();+ let lowercase: CharString = normalized.to_lower();self.word_map.get(normalized.as_ref())@@ -147,7 +147,7 @@ impl Dictionary for FullDictionary {fn contains_word(&self, word: &[char]) -> bool {let normalized = seq_to_normalized(word);- let lowercase: CharString = normalized.iter().flat_map(|c| c.to_lowercase()).collect();+ let lowercase: CharString = normalized.to_lower();self.word_map.contains_key(normalized.as_ref()) || self.word_map.contains_key(&lowercase)}@@ -173,14 +173,7 @@ impl Dictionary for FullDictionary {max_results: usize,) -> Vec<(&[char], u8, WordMetadata)> {let misspelled_normalized = seq_to_normalized(word);- let misspelled_lower: Vec<char> = misspelled_normalized- .iter()- .flat_map(|v| v.to_lowercase())- .collect();-- // 53 is the length of the longest word.- let mut buf_a = Vec::with_capacity(53);- let mut buf_b = Vec::with_capacity(53);+ let misspelled_lower: Vec<char> = misspelled_normalized.to_lower().to_vec();// The length of the shortest word to look at.let shortest_word_len = if misspelled_normalized.len() < max_distance as usize {@@ -195,14 +188,19 @@ impl Dictionary for FullDictionary {.rev().flat_map(|len| self.words_with_len_iter(len));+ // Pre-allocated vectors for the edit-distance calculation+ // 53 is the length of the longest word.+ let mut buf_a = Vec::with_capacity(53);+ let mut buf_b = Vec::with_capacity(53);let pruned_words = words_to_search.filter_map(|word| {let dist =edit_distance_min_alloc(&misspelled_normalized, word, &mut buf_a, &mut buf_b);let dist_lower =edit_distance_min_alloc(&misspelled_lower, word, &mut buf_a, &mut buf_b);+ let smallest_dist = std::cmp::min(dist, dist_lower);- if dist.min(dist_lower) <= max_distance {- Some((word, dist))+ if smallest_dist <= max_distance {+ Some((word, smallest_dist))} else {None}@@ -210,7 +208,6 @@ impl Dictionary for FullDictionary {// Locate the words with the lowest edit distance.let mut found_dist: Vec<(&[char], u8)> = Vec::with_capacity(max_results);-for (word, dist) in pruned_words {if found_dist.len() < max_results {found_dist.push((word, dist));@@ -223,7 +220,7 @@ impl Dictionary for FullDictionary {// Create final, ordered list of suggestions.found_dist.into_iter()- .map(|v| (v.0, v.1, self.get_word_metadata(v.0)))+ .map(|(word, dist)| (word, dist, self.get_word_metadata(word))).collect()}@@ -257,55 +254,11 @@ impl Dictionary for FullDictionary {}}-// Computes the Levenstein edit distance between two patterns.-// This is accomplished via a memory-optimized Wagner-Fischer algorithm-//-// This variant avoids allocation if you already have buffers.-#[inline]-pub fn edit_distance_min_alloc(- source: &[char],- target: &[char],- previous_row: &mut Vec<u8>,- current_row: &mut Vec<u8>,-) -> u8 {- if cfg!(debug_assertions) {- assert!(source.len() <= 255 && target.len() <= 255);- }-- let row_width = source.len();- let col_height = target.len();-- previous_row.clear();- previous_row.extend(0u8..=row_width as u8);- // Alright if not zeroed, since we overwrite it anyway.- current_row.resize(row_width + 1, 0);-- for j in 1..=col_height {- current_row[0] = j as u8;-- for i in 1..=row_width {- let cost = if source[i - 1] == target[j - 1] { 0 } else { 1 };-- current_row[i] = (previous_row[i] + 1)- .min(current_row[i - 1] + 1)- .min(previous_row[i - 1] + cost);- }-- std::mem::swap(previous_row, current_row);- }-- previous_row[row_width]-}--pub fn edit_distance(source: &[char], target: &[char]) -> u8 {- edit_distance_min_alloc(source, target, &mut Vec::new(), &mut Vec::new())-}-#[cfg(test)]mod tests {- use super::{edit_distance, seq_to_normalized};use itertools::Itertools;+ use super::super::{edit_distance, seq_to_normalized};use crate::{Dictionary, FullDictionary};fn assert_edit_dist(source: &str, target: &str, expected: u8) {diff --git a/harper-core/src/spell/mod.rs b/harper-core/src/spell/mod.rsindex 994cfab29..ca2439aaa 100644--- a/harper-core/src/spell/mod.rs+++ b/harper-core/src/spell/mod.rs@@ -1,8 +1,9 @@use std::borrow::Cow;+use harper_dictionary_parsing::WordMetadata;use itertools::{Itertools, MinMaxResult};-use crate::CharString;+use crate::{CharString, CharStringExt};pub use self::dictionary::Dictionary;pub use self::fst_dictionary::FstDictionary;@@ -22,16 +23,17 @@ pub fn suggest_correct_spelling<'a>(max_edit_dist: u8,dictionary: &'a impl Dictionary,) -> Vec<&'a [char]> {- let matches: Vec<&[char]> = dictionary+ let matches: Vec<(&[char], u8, WordMetadata)> = dictionary.fuzzy_match(misspelled_word, max_edit_dist, result_limit).into_iter()- .map(|r| r.0).collect();- let mut found = Vec::with_capacity(matches.len());+ let mut found: Vec<(&[char], u8, WordMetadata)> = Vec::with_capacity(matches.len());// Often the longest and the shortest words are the most helpful, so lets push// them first.- let minmax = matches.iter().position_minmax_by_key(|a| a.len());+ let minmax = matches+ .iter()+ .position_minmax_by_key(|(word, _, _)| word.len());if let MinMaxResult::MinMax(a, b) = minmax {if a == b {found.push(matches[a]);@@ -49,7 +51,6 @@ pub fn suggest_correct_spelling<'a>(.map(|v| v.1),);} else {- // Push the restfound.extend(matches);}@@ -59,15 +60,8 @@ pub fn suggest_correct_spelling<'a>(}// Let common words bubble up.- found.sort_by_key(|v| {- if dictionary.get_word_metadata(v).common {- 0- } else {- 1- }- });-- found+ found.sort_by_key(|(_, _, metadata)| if metadata.common { 0 } else { 1 });+ found.into_iter().map(|(word, _, _)| word).collect()}/// Convenience function over [`suggest_correct_spelling`] that does conversions@@ -81,13 +75,13 @@ pub fn suggest_correct_spelling_str(let chars: CharString = misspelled_word.into().chars().collect();suggest_correct_spelling(&chars, result_limit, max_edit_dist, dictionary).into_iter()- .map(|a| a.iter().collect::<String>())+ .map(|a| a.to_string()).collect()}/// Convert a given character sequence to the standard character set/// the dictionary is in.-pub fn seq_to_normalized(seq: &[char]) -> Cow<'_, [char]> {+pub(self) fn seq_to_normalized(seq: &[char]) -> Cow<'_, [char]> {if seq.iter().any(|c| char_to_normalized(*c) != *c) {Cow::Owned(seq.iter().copied().map(char_to_normalized).collect())} else {@@ -95,13 +89,57 @@ pub fn seq_to_normalized(seq: &[char]) -> Cow<'_, [char]> {}}-pub fn char_to_normalized(c: char) -> char {+pub(self) fn char_to_normalized(c: char) -> char {match c {'’' => '\'',_ => c,}}+// Computes the Levenshtein edit distance between two patterns.+// This is accomplished via a memory-optimized Wagner-Fischer algorithm+//+// This variant avoids allocation if you already have buffers.+#[inline]+pub(self) fn edit_distance_min_alloc(+ source: &[char],+ target: &[char],+ previous_row: &mut Vec<u8>,+ current_row: &mut Vec<u8>,+) -> u8 {+ if cfg!(debug_assertions) {+ assert!(source.len() <= 255 && target.len() <= 255);+ }++ let row_width = source.len();+ let col_height = target.len();++ previous_row.clear();+ previous_row.extend(0u8..=row_width as u8);+ // Alright if not zeroed, since we overwrite it anyway.+ current_row.resize(row_width + 1, 0);++ for j in 1..=col_height {+ current_row[0] = j as u8;++ for i in 1..=row_width {+ let cost = if source[i - 1] == target[j - 1] { 0 } else { 1 };++ current_row[i] = (previous_row[i] + 1)+ .min(current_row[i - 1] + 1)+ .min(previous_row[i - 1] + cost);+ }++ std::mem::swap(previous_row, current_row);+ }++ previous_row[row_width]+}++pub(self) fn edit_distance(source: &[char], target: &[char]) -> u8 {+ edit_distance_min_alloc(source, target, &mut Vec::new(), &mut Vec::new())+}+#[cfg(test)]mod tests {use itertools::Itertools;@@ -119,7 +157,7 @@ mod tests {#[test]fn issue_182() {- let results = suggest_correct_spelling_str("im", 100, 3, &FstDictionary::curated());+ let results = suggest_correct_spelling_str("im", 100, 3, &super::FullDictionary::curated());dbg!(&results);