Skip to content

Offline, privacy-first grammar checker. Fast, open-source, Rust-powered

feat(fst-dict): write some tests and change fst fuzzy search to order by edit distance after search

Grant Lemons5f42ba33c258
diff --git a/harper-core/src/char_string.rs b/harper-core/src/char_string.rs
index 3a30889c6..899f5bff8 100644
--- a/harper-core/src/char_string.rs
+++ b/harper-core/src/char_string.rs
@@ -6,6 +6,7 @@ pub type CharString = SmallVec<[char; 12]>;
pub trait CharStringExt {
fn to_lower(&self) -> CharString;
+ fn to_string(&self) -> String;
}
impl CharStringExt for [char] {
@@ -16,4 +17,7 @@ impl CharStringExt for [char] {
out
}
+ fn to_string(&self) -> String {
+ self.iter().collect()
+ }
}
diff --git a/harper-core/src/spell/fst_dictionary.rs b/harper-core/src/spell/fst_dictionary.rs
index 5a69d1116..68f2d0b14 100644
--- a/harper-core/src/spell/fst_dictionary.rs
+++ b/harper-core/src/spell/fst_dictionary.rs
@@ -1,8 +1,9 @@
-use super::{seq_to_normalized, FullDictionary};
+use super::{edit_distance_min_alloc, seq_to_normalized, FullDictionary};
use fst::{automaton::Levenshtein, IntoStreamer};
use fst::{Map as FstMap, Streamer};
+use itertools::Itertools;
-use crate::{Lrc, WordMetadata};
+use crate::{CharStringExt, Lrc, WordMetadata};
use super::Dictionary;
@@ -76,49 +77,74 @@ impl Dictionary for FstDictionary {
max_distance: u8,
max_results: usize,
) -> Vec<(&[char], u8, WordMetadata)> {
+ // Various transformations of the input
let chars: Vec<_> = word.chars().collect();
- let misspelled_normalized = seq_to_normalized(&chars);
- let misspelled_word: String = misspelled_normalized.iter().collect();
- let misspelled_lower: String = misspelled_normalized
- .iter()
- .flat_map(|v| v.to_lowercase())
- .collect();
-
- let aut = Levenshtein::new(&misspelled_word, max_distance as u32).unwrap();
- let aut_lower = Levenshtein::new(&misspelled_lower, max_distance as u32).unwrap();
- let mut word_lower_indexes_stream = self.word_map.search(aut_lower).into_stream();
- let mut word_indexes_stream = self.word_map.search(aut).into_stream();
-
- let mut word_indexes = Vec::with_capacity(max_results);
- let mut i = 0;
- while i < max_results {
+ let misspelled_word_charslice = seq_to_normalized(&chars);
+ let misspelled_lower_charslice = misspelled_word_charslice.to_lower();
+ let misspelled_word_string = misspelled_word_charslice.to_string();
+ let misspelled_lower_string = misspelled_lower_charslice.to_string();
+
+ // Actual FST search
+ let automaton = Levenshtein::new(&misspelled_word_string, max_distance as u32).unwrap();
+ let automaton_lower =
+ Levenshtein::new(&misspelled_lower_string, max_distance as u32).unwrap();
+ let mut word_lower_indexes_stream = self.word_map.search(automaton_lower).into_stream();
+ let mut word_indexes_stream = self.word_map.search(automaton).into_stream();
+
+ // Consume at most max_results values from each stream
+ // The search itself happens as you consume from the stream, so consuming a smaller number
+ // is more efficient
+ // HashSet used to dedup results
+ let mut words = hashbrown::HashSet::with_capacity(max_results * 2);
+ for _ in 0..max_results {
let mut flag = false;
- if let Some(v) = word_lower_indexes_stream.next() {
- word_indexes.push(v.1);
+ if let Some((_, index)) = word_lower_indexes_stream.next() {
+ words.insert(index);
flag = true;
}
- if let Some(v) = word_indexes_stream.next() {
- word_indexes.push(v.1);
+ if let Some((_, index)) = word_indexes_stream.next() {
+ words.insert(index);
flag = true;
}
if !flag {
break;
}
- i += 1;
}
- // Dedup but preserve order of word indexes
- let mut seen = hashbrown::HashSet::new();
- word_indexes.retain(|item| seen.insert(*item));
- word_indexes
+ // Pre-allocated vectors for edit-distance calculation
+ // 53 is the length of the longest word.
+ let mut buf_a = Vec::with_capacity(53);
+ let mut buf_b = Vec::with_capacity(53);
+
+ words
.into_iter()
+ .map(|index| (self.full_dict.get_word(index as usize), index))
+ // Sort by edit distance
+ .map(|(word, index)| {
+ let dist = edit_distance_min_alloc(
+ &misspelled_word_charslice,
+ word,
+ &mut buf_a,
+ &mut buf_b,
+ );
+ buf_a.clear();
+ buf_b.clear();
+ let dist_lower = edit_distance_min_alloc(
+ &misspelled_lower_charslice,
+ word,
+ &mut buf_a,
+ &mut buf_b,
+ );
+
+ (std::cmp::min(dist, dist_lower), word, index)
+ })
+ .sorted_unstable_by_key(|a| a.0)
.take(max_results)
- .map(|i| (self.full_dict.get_word(i as usize), i))
- .map(|(word, i)| {
+ .map(|(dist, word, index)| {
(
word.as_slice(),
- i as u8,
- self.full_dict.get_metadata(i as usize).to_owned(),
+ dist,
+ self.full_dict.get_metadata(index as usize).to_owned(),
)
})
.collect()
@@ -135,7 +161,10 @@ impl Dictionary for FstDictionary {
#[cfg(test)]
mod tests {
- use crate::{spell::seq_to_normalized, Dictionary};
+ use fst::IntoStreamer;
+ use itertools::Itertools;
+
+ use crate::{spell::seq_to_normalized, CharStringExt, Dictionary};
use super::FstDictionary;
@@ -145,11 +174,8 @@ mod tests {
for word in dict.words_iter() {
let misspelled_normalized = seq_to_normalized(word);
- let misspelled_word: String = misspelled_normalized.iter().collect();
- let misspelled_lower: String = misspelled_normalized
- .iter()
- .flat_map(|v| v.to_lowercase())
- .collect();
+ let misspelled_word: String = misspelled_normalized.to_string();
+ let misspelled_lower: String = misspelled_normalized.to_lower().to_string();
assert!(!misspelled_word.is_empty());
assert!(
@@ -165,11 +191,8 @@ mod tests {
let word: Vec<_> = "hello".chars().collect();
let misspelled_normalized = seq_to_normalized(&word);
- let misspelled_word: String = misspelled_normalized.iter().collect();
- let misspelled_lower: String = misspelled_normalized
- .iter()
- .flat_map(|v| v.to_lowercase())
- .collect();
+ let misspelled_word: String = misspelled_normalized.to_string();
+ let misspelled_lower: String = misspelled_normalized.to_lower().to_string();
assert!(dict.contains_word(&misspelled_normalized));
assert!(
@@ -177,4 +200,49 @@ mod tests {
|| dict.word_map.contains_key(misspelled_word)
);
}
+
+ #[test]
+ fn fst_search_hello() {
+ let dict = FstDictionary::curated();
+
+ let word: Vec<_> = "hvllo".chars().collect();
+ let misspelled_normalized = seq_to_normalized(&word);
+ let misspelled_word: String = misspelled_normalized.to_string();
+ let misspelled_lower: String = misspelled_normalized.to_lower().to_string();
+
+ let aut = fst::automaton::Levenshtein::new(&misspelled_word, 2).unwrap();
+ let aut_lower = fst::automaton::Levenshtein::new(&misspelled_lower, 2).unwrap();
+ let word_indexes_stream = dict
+ .word_map
+ .search(aut)
+ .into_stream()
+ .into_str_keys()
+ .unwrap();
+ let word_lower_indexes_stream = dict
+ .word_map
+ .search(aut_lower)
+ .into_stream()
+ .into_str_keys()
+ .unwrap();
+
+ dbg!(&word_indexes_stream);
+ assert!(
+ word_indexes_stream.contains(&"hello".to_string())
+ || word_lower_indexes_stream.contains(&"hello".to_string())
+ );
+ }
+
+ #[test]
+ fn fuzzy_hello() {
+ let dict = FstDictionary::curated();
+
+ let matches: Vec<_> = dict
+ .fuzzy_match_str("hvllo", 3, 100)
+ .iter()
+ .map(|cs| cs.0.iter().collect::<String>())
+ .collect();
+ dbg!(&matches);
+
+ assert!(matches.iter().take(10).contains(&"hello".to_string()))
+ }
}
diff --git a/harper-core/src/spell/full_dictionary.rs b/harper-core/src/spell/full_dictionary.rs
index b6af4063b..657a92e21 100644
--- a/harper-core/src/spell/full_dictionary.rs
+++ b/harper-core/src/spell/full_dictionary.rs
@@ -1,9 +1,9 @@
-use super::seq_to_normalized;
+use super::{edit_distance_min_alloc, seq_to_normalized};
use hashbrown::HashMap;
use smallvec::{SmallVec, ToSmallVec};
use super::dictionary::Dictionary;
-use crate::{CharString, Lrc, WordMetadata};
+use crate::{CharString, CharStringExt, Lrc, WordMetadata};
use harper_dictionary_parsing::{parse_default_attribute_list, parse_default_word_list};
/// A full, fat dictionary.
@@ -136,7 +136,7 @@ impl Default for FullDictionary {
impl Dictionary for FullDictionary {
fn get_word_metadata(&self, word: &[char]) -> WordMetadata {
let normalized = seq_to_normalized(word);
- let lowercase: CharString = normalized.iter().flat_map(|c| c.to_lowercase()).collect();
+ let lowercase: CharString = normalized.to_lower();
self.word_map
.get(normalized.as_ref())
@@ -147,7 +147,7 @@ impl Dictionary for FullDictionary {
fn contains_word(&self, word: &[char]) -> bool {
let normalized = seq_to_normalized(word);
- let lowercase: CharString = normalized.iter().flat_map(|c| c.to_lowercase()).collect();
+ let lowercase: CharString = normalized.to_lower();
self.word_map.contains_key(normalized.as_ref()) || self.word_map.contains_key(&lowercase)
}
@@ -173,14 +173,7 @@ impl Dictionary for FullDictionary {
max_results: usize,
) -> Vec<(&[char], u8, WordMetadata)> {
let misspelled_normalized = seq_to_normalized(word);
- let misspelled_lower: Vec<char> = misspelled_normalized
- .iter()
- .flat_map(|v| v.to_lowercase())
- .collect();
-
- // 53 is the length of the longest word.
- let mut buf_a = Vec::with_capacity(53);
- let mut buf_b = Vec::with_capacity(53);
+ let misspelled_lower: Vec<char> = misspelled_normalized.to_lower().to_vec();
// The length of the shortest word to look at.
let shortest_word_len = if misspelled_normalized.len() < max_distance as usize {
@@ -195,14 +188,19 @@ impl Dictionary for FullDictionary {
.rev()
.flat_map(|len| self.words_with_len_iter(len));
+ // Pre-allocated vectors for the edit-distance calculation
+ // 53 is the length of the longest word.
+ let mut buf_a = Vec::with_capacity(53);
+ let mut buf_b = Vec::with_capacity(53);
let pruned_words = words_to_search.filter_map(|word| {
let dist =
edit_distance_min_alloc(&misspelled_normalized, word, &mut buf_a, &mut buf_b);
let dist_lower =
edit_distance_min_alloc(&misspelled_lower, word, &mut buf_a, &mut buf_b);
+ let smallest_dist = std::cmp::min(dist, dist_lower);
- if dist.min(dist_lower) <= max_distance {
- Some((word, dist))
+ if smallest_dist <= max_distance {
+ Some((word, smallest_dist))
} else {
None
}
@@ -210,7 +208,6 @@ impl Dictionary for FullDictionary {
// Locate the words with the lowest edit distance.
let mut found_dist: Vec<(&[char], u8)> = Vec::with_capacity(max_results);
-
for (word, dist) in pruned_words {
if found_dist.len() < max_results {
found_dist.push((word, dist));
@@ -223,7 +220,7 @@ impl Dictionary for FullDictionary {
// Create final, ordered list of suggestions.
found_dist
.into_iter()
- .map(|v| (v.0, v.1, self.get_word_metadata(v.0)))
+ .map(|(word, dist)| (word, dist, self.get_word_metadata(word)))
.collect()
}
@@ -257,55 +254,11 @@ impl Dictionary for FullDictionary {
}
}
-// Computes the Levenstein edit distance between two patterns.
-// This is accomplished via a memory-optimized Wagner-Fischer algorithm
-//
-// This variant avoids allocation if you already have buffers.
-#[inline]
-pub fn edit_distance_min_alloc(
- source: &[char],
- target: &[char],
- previous_row: &mut Vec<u8>,
- current_row: &mut Vec<u8>,
-) -> u8 {
- if cfg!(debug_assertions) {
- assert!(source.len() <= 255 && target.len() <= 255);
- }
-
- let row_width = source.len();
- let col_height = target.len();
-
- previous_row.clear();
- previous_row.extend(0u8..=row_width as u8);
- // Alright if not zeroed, since we overwrite it anyway.
- current_row.resize(row_width + 1, 0);
-
- for j in 1..=col_height {
- current_row[0] = j as u8;
-
- for i in 1..=row_width {
- let cost = if source[i - 1] == target[j - 1] { 0 } else { 1 };
-
- current_row[i] = (previous_row[i] + 1)
- .min(current_row[i - 1] + 1)
- .min(previous_row[i - 1] + cost);
- }
-
- std::mem::swap(previous_row, current_row);
- }
-
- previous_row[row_width]
-}
-
-pub fn edit_distance(source: &[char], target: &[char]) -> u8 {
- edit_distance_min_alloc(source, target, &mut Vec::new(), &mut Vec::new())
-}
-
#[cfg(test)]
mod tests {
- use super::{edit_distance, seq_to_normalized};
use itertools::Itertools;
+ use super::super::{edit_distance, seq_to_normalized};
use crate::{Dictionary, FullDictionary};
fn assert_edit_dist(source: &str, target: &str, expected: u8) {
diff --git a/harper-core/src/spell/mod.rs b/harper-core/src/spell/mod.rs
index 994cfab29..ca2439aaa 100644
--- a/harper-core/src/spell/mod.rs
+++ b/harper-core/src/spell/mod.rs
@@ -1,8 +1,9 @@
use std::borrow::Cow;
+use harper_dictionary_parsing::WordMetadata;
use itertools::{Itertools, MinMaxResult};
-use crate::CharString;
+use crate::{CharString, CharStringExt};
pub use self::dictionary::Dictionary;
pub use self::fst_dictionary::FstDictionary;
@@ -22,16 +23,17 @@ pub fn suggest_correct_spelling<'a>(
max_edit_dist: u8,
dictionary: &'a impl Dictionary,
) -> Vec<&'a [char]> {
- let matches: Vec<&[char]> = dictionary
+ let matches: Vec<(&[char], u8, WordMetadata)> = dictionary
.fuzzy_match(misspelled_word, max_edit_dist, result_limit)
.into_iter()
- .map(|r| r.0)
.collect();
- let mut found = Vec::with_capacity(matches.len());
+ let mut found: Vec<(&[char], u8, WordMetadata)> = Vec::with_capacity(matches.len());
// Often the longest and the shortest words are the most helpful, so lets push
// them first.
- let minmax = matches.iter().position_minmax_by_key(|a| a.len());
+ let minmax = matches
+ .iter()
+ .position_minmax_by_key(|(word, _, _)| word.len());
if let MinMaxResult::MinMax(a, b) = minmax {
if a == b {
found.push(matches[a]);
@@ -49,7 +51,6 @@ pub fn suggest_correct_spelling<'a>(
.map(|v| v.1),
);
} else {
- // Push the rest
found.extend(matches);
}
@@ -59,15 +60,8 @@ pub fn suggest_correct_spelling<'a>(
}
// Let common words bubble up.
- found.sort_by_key(|v| {
- if dictionary.get_word_metadata(v).common {
- 0
- } else {
- 1
- }
- });
-
- found
+ found.sort_by_key(|(_, _, metadata)| if metadata.common { 0 } else { 1 });
+ found.into_iter().map(|(word, _, _)| word).collect()
}
/// Convenience function over [`suggest_correct_spelling`] that does conversions
@@ -81,13 +75,13 @@ pub fn suggest_correct_spelling_str(
let chars: CharString = misspelled_word.into().chars().collect();
suggest_correct_spelling(&chars, result_limit, max_edit_dist, dictionary)
.into_iter()
- .map(|a| a.iter().collect::<String>())
+ .map(|a| a.to_string())
.collect()
}
/// Convert a given character sequence to the standard character set
/// the dictionary is in.
-pub fn seq_to_normalized(seq: &[char]) -> Cow<'_, [char]> {
+pub(self) fn seq_to_normalized(seq: &[char]) -> Cow<'_, [char]> {
if seq.iter().any(|c| char_to_normalized(*c) != *c) {
Cow::Owned(seq.iter().copied().map(char_to_normalized).collect())
} else {
@@ -95,13 +89,57 @@ pub fn seq_to_normalized(seq: &[char]) -> Cow<'_, [char]> {
}
}
-pub fn char_to_normalized(c: char) -> char {
+pub(self) fn char_to_normalized(c: char) -> char {
match c {
'’' => '\'',
_ => c,
}
}
+// Computes the Levenshtein edit distance between two patterns.
+// This is accomplished via a memory-optimized Wagner-Fischer algorithm
+//
+// This variant avoids allocation if you already have buffers.
+#[inline]
+pub(self) fn edit_distance_min_alloc(
+ source: &[char],
+ target: &[char],
+ previous_row: &mut Vec<u8>,
+ current_row: &mut Vec<u8>,
+) -> u8 {
+ if cfg!(debug_assertions) {
+ assert!(source.len() <= 255 && target.len() <= 255);
+ }
+
+ let row_width = source.len();
+ let col_height = target.len();
+
+ previous_row.clear();
+ previous_row.extend(0u8..=row_width as u8);
+ // Alright if not zeroed, since we overwrite it anyway.
+ current_row.resize(row_width + 1, 0);
+
+ for j in 1..=col_height {
+ current_row[0] = j as u8;
+
+ for i in 1..=row_width {
+ let cost = if source[i - 1] == target[j - 1] { 0 } else { 1 };
+
+ current_row[i] = (previous_row[i] + 1)
+ .min(current_row[i - 1] + 1)
+ .min(previous_row[i - 1] + cost);
+ }
+
+ std::mem::swap(previous_row, current_row);
+ }
+
+ previous_row[row_width]
+}
+
+pub(self) fn edit_distance(source: &[char], target: &[char]) -> u8 {
+ edit_distance_min_alloc(source, target, &mut Vec::new(), &mut Vec::new())
+}
+
#[cfg(test)]
mod tests {
use itertools::Itertools;
@@ -119,7 +157,7 @@ mod tests {
#[test]
fn issue_182() {
- let results = suggest_correct_spelling_str("im", 100, 3, &FstDictionary::curated());
+ let results = suggest_correct_spelling_str("im", 100, 3, &super::FullDictionary::curated());
dbg!(&results);