Offline, privacy-first grammar checker. Fast, open-source, Rust-powered
feat(fst-dict): change to the better levenshtein automata implementation
Grant Lemons442010edb26e
diff --git a/Cargo.lock b/Cargo.lockindex ce0d75344..0f351b1f6 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -605,6 +605,7 @@ dependencies = ["hashbrown 0.15.0","is-macro","itertools 0.13.0",+ "levenshtein_automata","ordered-float","paste","pulldown-cmark",@@ -958,6 +959,15 @@ version = "1.5.0"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe"+[[package]]+name = "levenshtein_automata"+version = "0.2.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "0c2cdeb66e45e9f36bfad5bbdb4d2384e70936afbee843c6f6543f0c551ebb25"+dependencies = [+ "fst",+]+[[package]]name = "libc"version = "0.2.161"diff --git a/harper-core/Cargo.toml b/harper-core/Cargo.tomlindex 3b78763c3..b08ad34c4 100644--- a/harper-core/Cargo.toml+++ b/harper-core/Cargo.toml@@ -27,6 +27,7 @@ smallvec = { version = "1.13.2", features = ["serde"] }thiserror = "1.0.67"unicode-blocks = "0.1.9"unicode-width = "0.2.0"+levenshtein_automata = { version = "0.2.1", features = ["fst_automaton"] }[dev-dependencies]criterion = { version = "0.5.1", features = ["plotters", "cargo_bench_support"], default-features = false}diff --git a/harper-core/src/spell/fst_dictionary.rs b/harper-core/src/spell/fst_dictionary.rsindex 3f8582a61..8c74c3351 100644--- a/harper-core/src/spell/fst_dictionary.rs+++ b/harper-core/src/spell/fst_dictionary.rs@@ -1,8 +1,8 @@-use super::{edit_distance_min_alloc, seq_to_normalized, FullDictionary};-use fst::Map as FstMap;-use fst::{automaton::Levenshtein, IntoStreamer};+use super::{seq_to_normalized, FullDictionary};+use fst::{map::StreamWithState, IntoStreamer, Map as FstMap, Streamer};use hashbrown::HashMap;use itertools::Itertools;+use levenshtein_automata::{LevenshteinAutomatonBuilder, DFA};use std::sync::Arc;use crate::{CharString, CharStringExt, WordMetadata};@@ -79,6 +79,15 @@ impl FstDictionary {}}+fn stream_distances_vec(stream: &mut StreamWithState<&DFA>, dfa: &DFA) -> Vec<(u64, u8)> {+ let mut word_indexes = Vec::new();+ while let Some((_, v, s)) = stream.next() {+ word_indexes.push((v, dfa.distance(s).to_u8()));+ }++ word_indexes+}+impl Dictionary for FstDictionary {fn contains_word(&self, word: &[char]) -> bool {self.full_dict.contains_word(word)@@ -114,39 +123,23 @@ impl Dictionary for FstDictionary {// Various transformations of the inputlet chars: Vec<_> = word.chars().collect();let misspelled_word_charslice = seq_to_normalized(&chars);- let misspelled_lower_charslice = misspelled_word_charslice.to_lower();let misspelled_word_string = misspelled_word_charslice.to_string();// Actual FST search- let automaton = Levenshtein::new(&misspelled_word_string, max_distance as u32).unwrap();- let word_indexes = self.word_map.search(automaton).into_stream().into_values();-- // Pre-allocated vectors for edit-distance calculation- // 53 is the length of the longest word.- let mut buf_a = Vec::with_capacity(53);- let mut buf_b = Vec::with_capacity(53);+ let automaton_builder = LevenshteinAutomatonBuilder::new(max_distance, true);+ let dfa = automaton_builder.build_dfa(&misspelled_word_string);+ let mut word_indexes_stream = self.word_map.search_with_state(&dfa).into_stream();- word_indexes+ stream_distances_vec(&mut word_indexes_stream, &dfa).into_iter().sorted_unstable()- .dedup()- .map(|index| (self.full_dict.get_word(index as usize), index))+ .dedup_by(|a, b| a.0 == b.0)// Sort by edit distance- .map(|(word, index)| {- let dist = edit_distance_min_alloc(- &misspelled_lower_charslice,- &word.to_lower(),- &mut buf_a,- &mut buf_b,- );-- (word, dist, index)- }).sorted_unstable_by_key(|a| a.1).take(max_results)- .map(|(word, dist, index)| {+ .map(|(index, dist)| {(- word.as_slice(),+ self.full_dict.get_word(index as usize).as_slice(),dist,self.full_dict.get_metadata(index as usize).to_owned(),)diff --git a/harper-core/src/spell/full_dictionary.rs b/harper-core/src/spell/full_dictionary.rsindex 978b0b9a6..6ffe33742 100644--- a/harper-core/src/spell/full_dictionary.rs+++ b/harper-core/src/spell/full_dictionary.rs@@ -178,12 +178,11 @@ impl Dictionary for FullDictionary {max_distance: u8,max_results: usize,) -> Vec<(&[char], u8, WordMetadata)> {- let misspelled_normalized = seq_to_normalized(word);- let misspelled_lower_charslice = misspelled_normalized.to_lower();+ let misspelled_charslice = seq_to_normalized(word);let shortest_word_len =- std::cmp::max(1, misspelled_normalized.len() - max_distance as usize);- let longest_word_len = misspelled_normalized.len() + max_distance as usize;+ std::cmp::max(1, misspelled_charslice.len() - max_distance as usize);+ let longest_word_len = misspelled_charslice.len() + max_distance as usize;// Get canidate wordslet words_to_search = (shortest_word_len..=longest_word_len)@@ -198,12 +197,8 @@ impl Dictionary for FullDictionary {// Sort by edit-distancewords_to_search.filter_map(|word| {- let dist = edit_distance_min_alloc(- &misspelled_lower_charslice,- &word.to_lower(),- &mut buf_a,- &mut buf_b,- );+ let dist =+ edit_distance_min_alloc(&misspelled_charslice, &word, &mut buf_a, &mut buf_b);if dist <= max_distance {Some((word, dist))