Skip to content

Offline, privacy-first grammar checker. Fast, open-source, Rust-powered

feat(fst-dict): change to the better levenshtein automata implementation

Grant Lemons442010edb26e
diff --git a/Cargo.lock b/Cargo.lock
index ce0d75344..0f351b1f6 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -605,6 +605,7 @@ dependencies = [
"hashbrown 0.15.0",
"is-macro",
"itertools 0.13.0",
+ "levenshtein_automata",
"ordered-float",
"paste",
"pulldown-cmark",
@@ -958,6 +959,15 @@ version = "1.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe"
+[[package]]
+name = "levenshtein_automata"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0c2cdeb66e45e9f36bfad5bbdb4d2384e70936afbee843c6f6543f0c551ebb25"
+dependencies = [
+ "fst",
+]
+
[[package]]
name = "libc"
version = "0.2.161"
diff --git a/harper-core/Cargo.toml b/harper-core/Cargo.toml
index 3b78763c3..b08ad34c4 100644
--- a/harper-core/Cargo.toml
+++ b/harper-core/Cargo.toml
@@ -27,6 +27,7 @@ smallvec = { version = "1.13.2", features = ["serde"] }
thiserror = "1.0.67"
unicode-blocks = "0.1.9"
unicode-width = "0.2.0"
+levenshtein_automata = { version = "0.2.1", features = ["fst_automaton"] }
[dev-dependencies]
criterion = { version = "0.5.1", features = ["plotters", "cargo_bench_support"], default-features = false}
diff --git a/harper-core/src/spell/fst_dictionary.rs b/harper-core/src/spell/fst_dictionary.rs
index 3f8582a61..8c74c3351 100644
--- a/harper-core/src/spell/fst_dictionary.rs
+++ b/harper-core/src/spell/fst_dictionary.rs
@@ -1,8 +1,8 @@
-use super::{edit_distance_min_alloc, seq_to_normalized, FullDictionary};
-use fst::Map as FstMap;
-use fst::{automaton::Levenshtein, IntoStreamer};
+use super::{seq_to_normalized, FullDictionary};
+use fst::{map::StreamWithState, IntoStreamer, Map as FstMap, Streamer};
use hashbrown::HashMap;
use itertools::Itertools;
+use levenshtein_automata::{LevenshteinAutomatonBuilder, DFA};
use std::sync::Arc;
use crate::{CharString, CharStringExt, WordMetadata};
@@ -79,6 +79,15 @@ impl FstDictionary {
}
}
+fn stream_distances_vec(stream: &mut StreamWithState<&DFA>, dfa: &DFA) -> Vec<(u64, u8)> {
+ let mut word_indexes = Vec::new();
+ while let Some((_, v, s)) = stream.next() {
+ word_indexes.push((v, dfa.distance(s).to_u8()));
+ }
+
+ word_indexes
+}
+
impl Dictionary for FstDictionary {
fn contains_word(&self, word: &[char]) -> bool {
self.full_dict.contains_word(word)
@@ -114,39 +123,23 @@ impl Dictionary for FstDictionary {
// Various transformations of the input
let chars: Vec<_> = word.chars().collect();
let misspelled_word_charslice = seq_to_normalized(&chars);
- let misspelled_lower_charslice = misspelled_word_charslice.to_lower();
let misspelled_word_string = misspelled_word_charslice.to_string();
// Actual FST search
- let automaton = Levenshtein::new(&misspelled_word_string, max_distance as u32).unwrap();
- let word_indexes = self.word_map.search(automaton).into_stream().into_values();
-
- // Pre-allocated vectors for edit-distance calculation
- // 53 is the length of the longest word.
- let mut buf_a = Vec::with_capacity(53);
- let mut buf_b = Vec::with_capacity(53);
+ let automaton_builder = LevenshteinAutomatonBuilder::new(max_distance, true);
+ let dfa = automaton_builder.build_dfa(&misspelled_word_string);
+ let mut word_indexes_stream = self.word_map.search_with_state(&dfa).into_stream();
- word_indexes
+ stream_distances_vec(&mut word_indexes_stream, &dfa)
.into_iter()
.sorted_unstable()
- .dedup()
- .map(|index| (self.full_dict.get_word(index as usize), index))
+ .dedup_by(|a, b| a.0 == b.0)
// Sort by edit distance
- .map(|(word, index)| {
- let dist = edit_distance_min_alloc(
- &misspelled_lower_charslice,
- &word.to_lower(),
- &mut buf_a,
- &mut buf_b,
- );
-
- (word, dist, index)
- })
.sorted_unstable_by_key(|a| a.1)
.take(max_results)
- .map(|(word, dist, index)| {
+ .map(|(index, dist)| {
(
- word.as_slice(),
+ self.full_dict.get_word(index as usize).as_slice(),
dist,
self.full_dict.get_metadata(index as usize).to_owned(),
)
diff --git a/harper-core/src/spell/full_dictionary.rs b/harper-core/src/spell/full_dictionary.rs
index 978b0b9a6..6ffe33742 100644
--- a/harper-core/src/spell/full_dictionary.rs
+++ b/harper-core/src/spell/full_dictionary.rs
@@ -178,12 +178,11 @@ impl Dictionary for FullDictionary {
max_distance: u8,
max_results: usize,
) -> Vec<(&[char], u8, WordMetadata)> {
- let misspelled_normalized = seq_to_normalized(word);
- let misspelled_lower_charslice = misspelled_normalized.to_lower();
+ let misspelled_charslice = seq_to_normalized(word);
let shortest_word_len =
- std::cmp::max(1, misspelled_normalized.len() - max_distance as usize);
- let longest_word_len = misspelled_normalized.len() + max_distance as usize;
+ std::cmp::max(1, misspelled_charslice.len() - max_distance as usize);
+ let longest_word_len = misspelled_charslice.len() + max_distance as usize;
// Get canidate words
let words_to_search = (shortest_word_len..=longest_word_len)
@@ -198,12 +197,8 @@ impl Dictionary for FullDictionary {
// Sort by edit-distance
words_to_search
.filter_map(|word| {
- let dist = edit_distance_min_alloc(
- &misspelled_lower_charslice,
- &word.to_lower(),
- &mut buf_a,
- &mut buf_b,
- );
+ let dist =
+ edit_distance_min_alloc(&misspelled_charslice, &word, &mut buf_a, &mut buf_b);
if dist <= max_distance {
Some((word, dist))