Offline, privacy-first grammar checker. Fast, open-source, Rust-powered
fix(fst-dict): FstDictionary not matching both normalized and lowercase words
Grant Lemons5a95aaf5fd21
diff --git a/harper-core/build.rs b/harper-core/build.rsindex 89a2ac16d..3e351edd7 100644--- a/harper-core/build.rs+++ b/harper-core/build.rs@@ -26,14 +26,14 @@ fn main() {// Using u64 shouldn't pose any issues since I don't think the English// language has that many wordswords.iter().enumerate().for_each(|(i, s)| {+ let word = s.iter().collect::<String>();build- .insert(s.iter().collect::<String>(), i as u64)- .unwrap()+ .insert(word, i as u64)+ .expect("Insertion not in lexicographical order!");});build.finish().expect("Unable to build map of dictionary!");println!("cargo::rerun-if-changed=build.rs");- println!("cargo::rerun-if-changed=dictionary.fst");println!("cargo::rerun-if-changed=../harper-dictionary-parsing/dictionary.dict");}diff --git a/harper-core/src/spell/fst_dictionary.rs b/harper-core/src/spell/fst_dictionary.rsindex 95a6b5f90..5a69d1116 100644--- a/harper-core/src/spell/fst_dictionary.rs+++ b/harper-core/src/spell/fst_dictionary.rs@@ -77,25 +77,39 @@ impl Dictionary for FstDictionary {max_results: usize,) -> Vec<(&[char], u8, WordMetadata)> {let chars: Vec<_> = word.chars().collect();- let misspelled_word = seq_to_normalized(&chars);- let misspelled_lower: String = misspelled_word+ let misspelled_normalized = seq_to_normalized(&chars);+ let misspelled_word: String = misspelled_normalized.iter().collect();+ let misspelled_lower: String = misspelled_normalized.iter().flat_map(|v| v.to_lowercase()).collect();- let aut = Levenshtein::new(&misspelled_lower, max_distance as u32).unwrap();+ let aut = Levenshtein::new(&misspelled_word, max_distance as u32).unwrap();+ let aut_lower = Levenshtein::new(&misspelled_lower, max_distance as u32).unwrap();+ let mut word_lower_indexes_stream = self.word_map.search(aut_lower).into_stream();let mut word_indexes_stream = self.word_map.search(aut).into_stream();- let mut word_indexes = Vec::with_capacity(max_results);+ let mut word_indexes = Vec::with_capacity(max_results);let mut i = 0;while i < max_results {+ let mut flag = false;+ if let Some(v) = word_lower_indexes_stream.next() {+ word_indexes.push(v.1);+ flag = true;+ }if let Some(v) = word_indexes_stream.next() {word_indexes.push(v.1);- } else {+ flag = true;+ }+ if !flag {break;}i += 1;}+ // Dedup but preserve order of word indexes+ let mut seen = hashbrown::HashSet::new();+ word_indexes.retain(|item| seen.insert(*item));+word_indexes.into_iter().take(max_results)@@ -121,8 +135,6 @@ impl Dictionary for FstDictionary {#[cfg(test)]mod tests {- use fst::IntoStreamer;-use crate::{spell::seq_to_normalized, Dictionary};use super::FstDictionary;@@ -132,32 +144,18 @@ mod tests {let dict = FstDictionary::curated();for word in dict.words_iter() {- let misspelled_word = seq_to_normalized(word);- let misspelled_lower: String = misspelled_word+ let misspelled_normalized = seq_to_normalized(word);+ let misspelled_word: String = misspelled_normalized.iter().collect();+ let misspelled_lower: String = misspelled_normalized.iter().flat_map(|v| v.to_lowercase()).collect();- println!("{}", misspelled_lower);- assert!(!misspelled_lower.is_empty());- assert!(dict.contains_word(word));- assert!(dict.word_map.contains_key(misspelled_lower));- }- }-- #[test]- fn fst_words_match() {- let dict = FstDictionary::curated();-- for (word, i) in dict.word_map.into_stream().into_str_vec().unwrap() {- let full_dict_word = dict- .full_dict- .get_word(i as usize)- .iter()- .collect::<String>();-- println!("({i}) \"{}\" == \"{}\"?", word, full_dict_word);- assert_eq!(word, full_dict_word);+ assert!(!misspelled_word.is_empty());+ assert!(+ dict.word_map.contains_key(misspelled_word)+ || dict.word_map.contains_key(misspelled_lower)+ );}}@@ -166,13 +164,17 @@ mod tests {let dict = FstDictionary::curated();let word: Vec<_> = "hello".chars().collect();- let misspelled_word = seq_to_normalized(&word);- let misspelled_lower: String = misspelled_word+ let misspelled_normalized = seq_to_normalized(&word);+ let misspelled_word: String = misspelled_normalized.iter().collect();+ let misspelled_lower: String = misspelled_normalized.iter().flat_map(|v| v.to_lowercase()).collect();- assert!(dict.contains_word(&word));- assert!(dict.word_map.contains_key(misspelled_lower));+ assert!(dict.contains_word(&misspelled_normalized));+ assert!(+ dict.word_map.contains_key(misspelled_lower)+ || dict.word_map.contains_key(misspelled_word)+ );}}diff --git a/harper-core/src/spell/full_dictionary.rs b/harper-core/src/spell/full_dictionary.rsindex 6662dfd97..b6af4063b 100644--- a/harper-core/src/spell/full_dictionary.rs+++ b/harper-core/src/spell/full_dictionary.rs@@ -172,8 +172,8 @@ impl Dictionary for FullDictionary {max_distance: u8,max_results: usize,) -> Vec<(&[char], u8, WordMetadata)> {- let misspelled_word = seq_to_normalized(word);- let misspelled_lower: Vec<char> = misspelled_word+ let misspelled_normalized = seq_to_normalized(word);+ let misspelled_lower: Vec<char> = misspelled_normalized.iter().flat_map(|v| v.to_lowercase()).collect();@@ -183,19 +183,21 @@ impl Dictionary for FullDictionary {let mut buf_b = Vec::with_capacity(53);// The length of the shortest word to look at.- let shortest_word_len = if misspelled_word.len() < max_distance as usize {+ let shortest_word_len = if misspelled_normalized.len() < max_distance as usize {1} else {- misspelled_word.len() - max_distance as usize+ misspelled_normalized.len() - max_distance as usize};// Note how we look at the biggest words first.- let words_to_search = (shortest_word_len..misspelled_word.len() + max_distance as usize)+ let words_to_search = (shortest_word_len+ ..misspelled_normalized.len() + max_distance as usize).rev().flat_map(|len| self.words_with_len_iter(len));let pruned_words = words_to_search.filter_map(|word| {- let dist = edit_distance_min_alloc(&misspelled_word, word, &mut buf_a, &mut buf_b);+ let dist =+ edit_distance_min_alloc(&misspelled_normalized, word, &mut buf_a, &mut buf_b);let dist_lower =edit_distance_min_alloc(&misspelled_lower, word, &mut buf_a, &mut buf_b);@@ -362,8 +364,8 @@ mod tests {#[test]fn herself_is_pronoun() {let dict = FullDictionary::curated();- assert!(dict.get_word_metadata_str("than").is_conjunction());- assert!(dict.get_word_metadata_str("Than").is_conjunction());+ assert!(dict.get_word_metadata_str("herself").is_pronoun());+ assert!(dict.get_word_metadata_str("Herself").is_pronoun());}#[test]