Skip to content

Offline, privacy-first grammar checker. Fast, open-source, Rust-powered

fix(fst-dict): make word len function not mutate word list and remove numbers from dictionary

Grant Lemons9e955b3fdada
diff --git a/harper-core/Cargo.toml b/harper-core/Cargo.toml
index 72bb57d9f..63951cc4b 100644
--- a/harper-core/Cargo.toml
+++ b/harper-core/Cargo.toml
@@ -7,6 +7,10 @@ license = "Apache-2.0"
readme = "README.md"
repository = "https://github.com/elijah-potter/harper"
+[[bin]]
+name = "force-build"
+path = "build.rs"
+
[dependencies]
harper-dictionary-parsing = { path = "../harper-dictionary-parsing", version = "0.12.0" }
blanket = "0.4.0"
diff --git a/harper-core/src/spell/fst_dictionary.rs b/harper-core/src/spell/fst_dictionary.rs
index 11168a692..f3274da1c 100644
--- a/harper-core/src/spell/fst_dictionary.rs
+++ b/harper-core/src/spell/fst_dictionary.rs
@@ -155,7 +155,8 @@ mod tests {
.get_word(i as usize)
.iter()
.collect::<String>();
- println!("\"{}\" == \"{}\"?", word, full_dict_word);
+
+ println!("({i}) \"{}\" == \"{}\"?", word, full_dict_word);
assert_eq!(word, full_dict_word);
}
}
diff --git a/harper-core/src/spell/full_dictionary.rs b/harper-core/src/spell/full_dictionary.rs
index ce5dbc97a..fca80dd8c 100644
--- a/harper-core/src/spell/full_dictionary.rs
+++ b/harper-core/src/spell/full_dictionary.rs
@@ -42,7 +42,7 @@ fn uncached_inner_new() -> Lrc<FullDictionary> {
Lrc::new(FullDictionary {
word_map,
- word_len_starts: FullDictionary::create_len_starts(&mut words),
+ word_len_starts: FullDictionary::create_len_starts(&words),
words,
})
}
@@ -79,7 +79,7 @@ impl FullDictionary {
.collect();
self.words.extend(pairs.iter().map(|(v, _)| v.clone()));
- self.word_len_starts = Self::create_len_starts(&mut self.words);
+ self.word_len_starts = Self::create_len_starts(&self.words);
self.word_map.extend(pairs);
}
@@ -103,11 +103,13 @@ impl FullDictionary {
/// list. NOTE: This function will sort the original word list by its
/// length. If the word list's order is changed after creating the
/// lookup, it will no longer be valid.
- fn create_len_starts(words: &mut [CharString]) -> Vec<usize> {
- words.sort_by_key(|a| a.len());
+ fn create_len_starts(words: &Vec<CharString>) -> Vec<usize> {
+ let mut len_words = words.clone();
+
+ len_words.sort_by_key(|a| a.len());
let mut word_len_starts = vec![0, 0];
- for (index, len) in words.iter().map(SmallVec::len).enumerate() {
+ for (index, len) in len_words.iter().map(SmallVec::len).enumerate() {
if word_len_starts.len() <= len {
word_len_starts.resize(len, index);
word_len_starts.push(index);
diff --git a/harper-dictionary-parsing/dictionary.dict b/harper-dictionary-parsing/dictionary.dict
index 57842e29b..f226f088f 100644
--- a/harper-dictionary-parsing/dictionary.dict
+++ b/harper-dictionary-parsing/dictionary.dict
@@ -1,27 +1,4 @@
50000
-0/1nm
-0th/5pt
-1/1n
-1st/5p
-1th/tc
-2/51nm
-2nd/5p
-2th/tc
-3/1nm
-3rd/54p
-3th/tc
-4/nm
-4th/52pt
-5/1nm
-5th/5pt
-6/1nm
-6th/5pt
-7/1nm
-7th/5pt
-8/4nm
-8th/5pt
-9/1nm
-9th/5pt
A/125SM
AA/1254M
AAA/12