Skip to content

Offline, privacy-first grammar checker. Fast, open-source, Rust-powered

feat(fst-dict): use a fulldictionary for all parts of fstdictionary except fuzzy search

Grant Lemons9879f5ad1f8a
diff --git a/Cargo.lock b/Cargo.lock
index da7e3c7c9..2f15086a3 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -304,6 +304,7 @@ dependencies = [
"num-traits",
"once_cell",
"oorandom",
+ "plotters",
"regex",
"serde",
"serde_derive",
@@ -1153,6 +1154,34 @@ version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184"
+[[package]]
+name = "plotters"
+version = "0.3.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5aeb6f403d7a4911efb1e33402027fc44f29b5bf6def3effcc22d7bb75f2b747"
+dependencies = [
+ "num-traits",
+ "plotters-backend",
+ "plotters-svg",
+ "wasm-bindgen",
+ "web-sys",
+]
+
+[[package]]
+name = "plotters-backend"
+version = "0.3.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "df42e13c12958a16b3f7f4386b9ab1f3e7933914ecea48da7139435263a4172a"
+
+[[package]]
+name = "plotters-svg"
+version = "0.3.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "51bae2ac328883f7acdfea3d66a7c35751187f870bc81f94563733a154d7a670"
+dependencies = [
+ "plotters-backend",
+]
+
[[package]]
name = "ppv-lite86"
version = "0.2.20"
@@ -1972,6 +2001,16 @@ version = "0.2.95"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "65fc09f10666a9f147042251e0dda9c18f166ff7de300607007e96bdebc1068d"
+[[package]]
+name = "web-sys"
+version = "0.3.72"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f6488b90108c040df0fe62fa815cbdee25124641df01814dd7282749234c6112"
+dependencies = [
+ "js-sys",
+ "wasm-bindgen",
+]
+
[[package]]
name = "winapi"
version = "0.3.9"
diff --git a/harper-cli/src/main.rs b/harper-cli/src/main.rs
index 0838058f3..73b7d45da 100644
--- a/harper-cli/src/main.rs
+++ b/harper-cli/src/main.rs
@@ -8,7 +8,7 @@ use clap::Parser;
use harper_comments::CommentParser;
use harper_core::linting::{LintGroup, LintGroupConfig, Linter};
use harper_core::parsers::Markdown;
-use harper_core::{remove_overlaps, Dictionary, Document, FullDictionary};
+use harper_core::{remove_overlaps, Dictionary, Document, FstDictionary};
#[derive(Debug, Parser)]
enum Args {
@@ -37,7 +37,7 @@ fn main() -> anyhow::Result<()> {
Args::Lint { file, count } => {
let (doc, source) = load_file(&file)?;
- let mut linter = LintGroup::new(LintGroupConfig::default(), FullDictionary::curated());
+ let mut linter = LintGroup::new(LintGroupConfig::default(), FstDictionary::curated());
let mut lints = linter.lint(&doc);
if count {
@@ -85,7 +85,7 @@ fn main() -> anyhow::Result<()> {
Ok(())
}
Args::Words => {
- let dict = FullDictionary::curated();
+ let dict = FstDictionary::curated();
let mut word_str = String::new();
diff --git a/harper-core/Cargo.toml b/harper-core/Cargo.toml
index ceeb30ffa..65344943c 100644
--- a/harper-core/Cargo.toml
+++ b/harper-core/Cargo.toml
@@ -24,7 +24,7 @@ unicode-blocks = "0.1.9"
unicode-width = "0.2.0"
[dev-dependencies]
-criterion = { version = "0.5.1", default-features = false }
+criterion = { version = "0.5.1", features = ["plotters", "cargo_bench_support"], default-features = false}
rand = "0.8.5"
[build-dependencies]
diff --git a/harper-core/src/spell/fst_dictionary.rs b/harper-core/src/spell/fst_dictionary.rs
index 6545f56f1..c0b23a3d5 100644
--- a/harper-core/src/spell/fst_dictionary.rs
+++ b/harper-core/src/spell/fst_dictionary.rs
@@ -1,18 +1,15 @@
-use fst::Map as FstMap;
+use super::FullDictionary;
use fst::{automaton::Levenshtein, IntoStreamer};
-use itertools::Itertools;
+use fst::{Map as FstMap, Streamer};
-use super::hunspell::{parse_default_attribute_list, parse_default_word_list};
-use crate::{CharString, Lrc, WordMetadata};
+use crate::{Lrc, WordMetadata};
use super::Dictionary;
#[derive(Debug)]
pub struct FstDictionary {
- /// Sorted by string in lexicographic order
- metadata: Vec<WordMetadata>,
- /// Sorted in lexicographic order
- words: Vec<CharString>,
+ /// Underlying FullDictionary used for everything except fuzzy finding
+ full_dict: Lrc<FullDictionary>,
/// Used for fuzzy-finding the index of words or metadata
word_map: FstMap<Vec<u8>>,
}
@@ -20,27 +17,11 @@ pub struct FstDictionary {
/// The uncached function that is used to produce the original copy of the
/// curated dictionary.
fn uncached_inner_new() -> Lrc<FstDictionary> {
- let word_list = parse_default_word_list().unwrap();
- let attr_list = parse_default_attribute_list();
-
- // There will be at _least_ this number of words
- // This creates a memory map, which enables searching the map without loading
- // all of it into memory.
+ let full_dict = FullDictionary::curated();
let word_map = FstMap::new(include_bytes!("../../dictionary.fst").to_vec()).unwrap();
- // There will be at _least_ this number of words
- let mut wmap = hashbrown::HashMap::with_capacity(word_list.len());
- attr_list.expand_marked_words(word_list, &mut wmap);
-
- let (words, metadata) = wmap
- .into_iter()
- .sorted_by(|a, b| Ord::cmp(a.0.as_slice(), b.0.as_slice()))
- .dedup_by(|a, b| a.0 == b.0)
- .unzip();
-
Lrc::new(FstDictionary {
- metadata,
- words,
+ full_dict,
word_map,
})
}
@@ -59,29 +40,19 @@ impl FstDictionary {
impl Dictionary for FstDictionary {
fn contains_word(&self, word: &[char]) -> bool {
- self.word_map.contains_key(word.iter().collect::<String>())
+ self.full_dict.contains_word(word)
}
fn contains_word_str(&self, word: &str) -> bool {
- self.word_map.contains_key(word)
+ self.full_dict.contains_word_str(word)
}
fn get_word_metadata(&self, word: &[char]) -> WordMetadata {
- let index: Option<usize> = self
- .word_map
- .get(word.iter().collect::<String>())
- .map(|i| i as usize);
-
- if let Some(i) = index {
- self.metadata[i]
- } else {
- WordMetadata::default()
- }
+ self.full_dict.get_word_metadata(word)
}
fn get_word_metadata_str(&self, word: &str) -> WordMetadata {
- let index: usize = self.word_map.get(word).unwrap() as usize;
- self.metadata[index]
+ self.full_dict.get_word_metadata_str(word)
}
fn fuzzy_match(
@@ -100,21 +71,36 @@ impl Dictionary for FstDictionary {
max_results: usize,
) -> Vec<(&[char], u8, WordMetadata)> {
let aut = Levenshtein::new(word, max_distance as u32).unwrap();
- let word_indexes: Vec<u64> = self.word_map.search(aut).into_stream().into_values();
+ let mut word_indexes_stream = self.word_map.search(aut).into_stream();
+ let mut word_indexes = Vec::with_capacity(max_results);
+
+ let i = 0;
+ while i < max_results {
+ if let Some(v) = word_indexes_stream.next() {
+ word_indexes.push(v.1);
+ } else {
+ break;
+ }
+ }
word_indexes
.into_iter()
.take(max_results)
- .map(|i| (self.words.get(i as usize).unwrap(), i))
- .map(|(word, i)| (word.as_slice(), i as u8, self.metadata[i as usize]))
+ .map(|i| (self.full_dict.get_word(i as usize), i))
+ .map(|(word, i)| {
+ (
+ word.as_slice(),
+ i as u8,
+ self.full_dict.get_metadata(i as usize).to_owned(),
+ )
+ })
.collect()
}
fn words_iter(&self) -> impl Iterator<Item = &'_ [char]> {
- self.words.iter().map(|v| v.as_slice())
+ self.full_dict.words_iter()
}
fn words_with_len_iter(&self, len: usize) -> Box<dyn Iterator<Item = &'_ [char]> + '_> {
- // Could be _way_ faster.
- Box::new(self.words_iter().filter(move |w| w.len() == len))
+ self.full_dict.words_with_len_iter(len)
}
}
diff --git a/harper-core/src/spell/full_dictionary.rs b/harper-core/src/spell/full_dictionary.rs
index 02945ce89..8cd5a85ad 100644
--- a/harper-core/src/spell/full_dictionary.rs
+++ b/harper-core/src/spell/full_dictionary.rs
@@ -1,5 +1,4 @@
-use std::borrow::Cow;
-
+use super::seq_to_normalized;
use hashbrown::HashMap;
use smallvec::{SmallVec, ToSmallVec};
@@ -117,6 +116,13 @@ impl FullDictionary {
word_len_starts
}
+
+ pub fn get_word(&self, index: usize) -> &CharString {
+ &self.words[index]
+ }
+ pub fn get_metadata(&self, index: usize) -> &WordMetadata {
+ self.word_map.get(self.get_word(index)).unwrap()
+ }
}
impl Default for FullDictionary {
@@ -291,23 +297,6 @@ pub fn edit_distance(source: &[char], target: &[char]) -> u8 {
edit_distance_min_alloc(source, target, &mut Vec::new(), &mut Vec::new())
}
-/// Convert a given character sequence to the standard character set
-/// the dictionary is in.
-pub fn seq_to_normalized(seq: &[char]) -> Cow<'_, [char]> {
- if seq.iter().any(|c| char_to_normalized(*c) != *c) {
- Cow::Owned(seq.iter().copied().map(char_to_normalized).collect())
- } else {
- Cow::Borrowed(seq)
- }
-}
-
-pub fn char_to_normalized(c: char) -> char {
- match c {
- '’' => '\'',
- _ => c,
- }
-}
-
#[cfg(test)]
mod tests {
use super::{edit_distance, seq_to_normalized};
diff --git a/harper-core/src/spell/mod.rs b/harper-core/src/spell/mod.rs
index f196fdb7f..38b3ecdbb 100644
--- a/harper-core/src/spell/mod.rs
+++ b/harper-core/src/spell/mod.rs
@@ -1,3 +1,5 @@
+use std::borrow::Cow;
+
use itertools::{Itertools, MinMaxResult};
use crate::CharString;
@@ -84,6 +86,23 @@ pub fn suggest_correct_spelling_str(
.collect()
}
+/// Convert a given character sequence to the standard character set
+/// the dictionary is in.
+pub fn seq_to_normalized(seq: &[char]) -> Cow<'_, [char]> {
+ if seq.iter().any(|c| char_to_normalized(*c) != *c) {
+ Cow::Owned(seq.iter().copied().map(char_to_normalized).collect())
+ } else {
+ Cow::Borrowed(seq)
+ }
+}
+
+pub fn char_to_normalized(c: char) -> char {
+ match c {
+ '’' => '\'',
+ _ => c,
+ }
+}
+
#[cfg(test)]
mod tests {
use itertools::Itertools;