Skip to content

Offline, privacy-first grammar checker. Fast, open-source, Rust-powered

fix(fst-dict): FstDictionary not matching both normalized and lowercase words

Grant Lemons5a95aaf5fd21
diff --git a/harper-core/build.rs b/harper-core/build.rs
index 89a2ac16d..3e351edd7 100644
--- a/harper-core/build.rs
+++ b/harper-core/build.rs
@@ -26,14 +26,14 @@ fn main() {
// Using u64 shouldn't pose any issues since I don't think the English
// language has that many words
words.iter().enumerate().for_each(|(i, s)| {
+ let word = s.iter().collect::<String>();
build
- .insert(s.iter().collect::<String>(), i as u64)
- .unwrap()
+ .insert(word, i as u64)
+ .expect("Insertion not in lexicographical order!");
});
build.finish().expect("Unable to build map of dictionary!");
println!("cargo::rerun-if-changed=build.rs");
- println!("cargo::rerun-if-changed=dictionary.fst");
println!("cargo::rerun-if-changed=../harper-dictionary-parsing/dictionary.dict");
}
diff --git a/harper-core/src/spell/fst_dictionary.rs b/harper-core/src/spell/fst_dictionary.rs
index 95a6b5f90..5a69d1116 100644
--- a/harper-core/src/spell/fst_dictionary.rs
+++ b/harper-core/src/spell/fst_dictionary.rs
@@ -77,25 +77,39 @@ impl Dictionary for FstDictionary {
max_results: usize,
) -> Vec<(&[char], u8, WordMetadata)> {
let chars: Vec<_> = word.chars().collect();
- let misspelled_word = seq_to_normalized(&chars);
- let misspelled_lower: String = misspelled_word
+ let misspelled_normalized = seq_to_normalized(&chars);
+ let misspelled_word: String = misspelled_normalized.iter().collect();
+ let misspelled_lower: String = misspelled_normalized
.iter()
.flat_map(|v| v.to_lowercase())
.collect();
- let aut = Levenshtein::new(&misspelled_lower, max_distance as u32).unwrap();
+ let aut = Levenshtein::new(&misspelled_word, max_distance as u32).unwrap();
+ let aut_lower = Levenshtein::new(&misspelled_lower, max_distance as u32).unwrap();
+ let mut word_lower_indexes_stream = self.word_map.search(aut_lower).into_stream();
let mut word_indexes_stream = self.word_map.search(aut).into_stream();
- let mut word_indexes = Vec::with_capacity(max_results);
+ let mut word_indexes = Vec::with_capacity(max_results);
let mut i = 0;
while i < max_results {
+ let mut flag = false;
+ if let Some(v) = word_lower_indexes_stream.next() {
+ word_indexes.push(v.1);
+ flag = true;
+ }
if let Some(v) = word_indexes_stream.next() {
word_indexes.push(v.1);
- } else {
+ flag = true;
+ }
+ if !flag {
break;
}
i += 1;
}
+ // Dedup but preserve order of word indexes
+ let mut seen = hashbrown::HashSet::new();
+ word_indexes.retain(|item| seen.insert(*item));
+
word_indexes
.into_iter()
.take(max_results)
@@ -121,8 +135,6 @@ impl Dictionary for FstDictionary {
#[cfg(test)]
mod tests {
- use fst::IntoStreamer;
-
use crate::{spell::seq_to_normalized, Dictionary};
use super::FstDictionary;
@@ -132,32 +144,18 @@ mod tests {
let dict = FstDictionary::curated();
for word in dict.words_iter() {
- let misspelled_word = seq_to_normalized(word);
- let misspelled_lower: String = misspelled_word
+ let misspelled_normalized = seq_to_normalized(word);
+ let misspelled_word: String = misspelled_normalized.iter().collect();
+ let misspelled_lower: String = misspelled_normalized
.iter()
.flat_map(|v| v.to_lowercase())
.collect();
- println!("{}", misspelled_lower);
- assert!(!misspelled_lower.is_empty());
- assert!(dict.contains_word(word));
- assert!(dict.word_map.contains_key(misspelled_lower));
- }
- }
-
- #[test]
- fn fst_words_match() {
- let dict = FstDictionary::curated();
-
- for (word, i) in dict.word_map.into_stream().into_str_vec().unwrap() {
- let full_dict_word = dict
- .full_dict
- .get_word(i as usize)
- .iter()
- .collect::<String>();
-
- println!("({i}) \"{}\" == \"{}\"?", word, full_dict_word);
- assert_eq!(word, full_dict_word);
+ assert!(!misspelled_word.is_empty());
+ assert!(
+ dict.word_map.contains_key(misspelled_word)
+ || dict.word_map.contains_key(misspelled_lower)
+ );
}
}
@@ -166,13 +164,17 @@ mod tests {
let dict = FstDictionary::curated();
let word: Vec<_> = "hello".chars().collect();
- let misspelled_word = seq_to_normalized(&word);
- let misspelled_lower: String = misspelled_word
+ let misspelled_normalized = seq_to_normalized(&word);
+ let misspelled_word: String = misspelled_normalized.iter().collect();
+ let misspelled_lower: String = misspelled_normalized
.iter()
.flat_map(|v| v.to_lowercase())
.collect();
- assert!(dict.contains_word(&word));
- assert!(dict.word_map.contains_key(misspelled_lower));
+ assert!(dict.contains_word(&misspelled_normalized));
+ assert!(
+ dict.word_map.contains_key(misspelled_lower)
+ || dict.word_map.contains_key(misspelled_word)
+ );
}
}
diff --git a/harper-core/src/spell/full_dictionary.rs b/harper-core/src/spell/full_dictionary.rs
index 6662dfd97..b6af4063b 100644
--- a/harper-core/src/spell/full_dictionary.rs
+++ b/harper-core/src/spell/full_dictionary.rs
@@ -172,8 +172,8 @@ impl Dictionary for FullDictionary {
max_distance: u8,
max_results: usize,
) -> Vec<(&[char], u8, WordMetadata)> {
- let misspelled_word = seq_to_normalized(word);
- let misspelled_lower: Vec<char> = misspelled_word
+ let misspelled_normalized = seq_to_normalized(word);
+ let misspelled_lower: Vec<char> = misspelled_normalized
.iter()
.flat_map(|v| v.to_lowercase())
.collect();
@@ -183,19 +183,21 @@ impl Dictionary for FullDictionary {
let mut buf_b = Vec::with_capacity(53);
// The length of the shortest word to look at.
- let shortest_word_len = if misspelled_word.len() < max_distance as usize {
+ let shortest_word_len = if misspelled_normalized.len() < max_distance as usize {
1
} else {
- misspelled_word.len() - max_distance as usize
+ misspelled_normalized.len() - max_distance as usize
};
// Note how we look at the biggest words first.
- let words_to_search = (shortest_word_len..misspelled_word.len() + max_distance as usize)
+ let words_to_search = (shortest_word_len
+ ..misspelled_normalized.len() + max_distance as usize)
.rev()
.flat_map(|len| self.words_with_len_iter(len));
let pruned_words = words_to_search.filter_map(|word| {
- let dist = edit_distance_min_alloc(&misspelled_word, word, &mut buf_a, &mut buf_b);
+ let dist =
+ edit_distance_min_alloc(&misspelled_normalized, word, &mut buf_a, &mut buf_b);
let dist_lower =
edit_distance_min_alloc(&misspelled_lower, word, &mut buf_a, &mut buf_b);
@@ -362,8 +364,8 @@ mod tests {
#[test]
fn herself_is_pronoun() {
let dict = FullDictionary::curated();
- assert!(dict.get_word_metadata_str("than").is_conjunction());
- assert!(dict.get_word_metadata_str("Than").is_conjunction());
+ assert!(dict.get_word_metadata_str("herself").is_pronoun());
+ assert!(dict.get_word_metadata_str("Herself").is_pronoun());
}
#[test]