Offline, privacy-first grammar checker. Fast, open-source, Rust-powered
feat(fst-dict): use a fulldictionary for all parts of fstdictionary except fuzzy search
Grant Lemons9879f5ad1f8a
diff --git a/Cargo.lock b/Cargo.lockindex da7e3c7c9..2f15086a3 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -304,6 +304,7 @@ dependencies = ["num-traits","once_cell","oorandom",+ "plotters","regex","serde","serde_derive",@@ -1153,6 +1154,34 @@ version = "0.1.0"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184"+[[package]]+name = "plotters"+version = "0.3.7"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "5aeb6f403d7a4911efb1e33402027fc44f29b5bf6def3effcc22d7bb75f2b747"+dependencies = [+ "num-traits",+ "plotters-backend",+ "plotters-svg",+ "wasm-bindgen",+ "web-sys",+]++[[package]]+name = "plotters-backend"+version = "0.3.7"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "df42e13c12958a16b3f7f4386b9ab1f3e7933914ecea48da7139435263a4172a"++[[package]]+name = "plotters-svg"+version = "0.3.7"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "51bae2ac328883f7acdfea3d66a7c35751187f870bc81f94563733a154d7a670"+dependencies = [+ "plotters-backend",+]+[[package]]name = "ppv-lite86"version = "0.2.20"@@ -1972,6 +2001,16 @@ version = "0.2.95"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "65fc09f10666a9f147042251e0dda9c18f166ff7de300607007e96bdebc1068d"+[[package]]+name = "web-sys"+version = "0.3.72"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "f6488b90108c040df0fe62fa815cbdee25124641df01814dd7282749234c6112"+dependencies = [+ "js-sys",+ "wasm-bindgen",+]+[[package]]name = "winapi"version = "0.3.9"diff --git a/harper-cli/src/main.rs b/harper-cli/src/main.rsindex 0838058f3..73b7d45da 100644--- a/harper-cli/src/main.rs+++ b/harper-cli/src/main.rs@@ -8,7 +8,7 @@ use clap::Parser;use harper_comments::CommentParser;use harper_core::linting::{LintGroup, LintGroupConfig, Linter};use harper_core::parsers::Markdown;-use harper_core::{remove_overlaps, Dictionary, Document, FullDictionary};+use harper_core::{remove_overlaps, Dictionary, Document, FstDictionary};#[derive(Debug, Parser)]enum Args {@@ -37,7 +37,7 @@ fn main() -> anyhow::Result<()> {Args::Lint { file, count } => {let (doc, source) = load_file(&file)?;- let mut linter = LintGroup::new(LintGroupConfig::default(), FullDictionary::curated());+ let mut linter = LintGroup::new(LintGroupConfig::default(), FstDictionary::curated());let mut lints = linter.lint(&doc);if count {@@ -85,7 +85,7 @@ fn main() -> anyhow::Result<()> {Ok(())}Args::Words => {- let dict = FullDictionary::curated();+ let dict = FstDictionary::curated();let mut word_str = String::new();diff --git a/harper-core/Cargo.toml b/harper-core/Cargo.tomlindex ceeb30ffa..65344943c 100644--- a/harper-core/Cargo.toml+++ b/harper-core/Cargo.toml@@ -24,7 +24,7 @@ unicode-blocks = "0.1.9"unicode-width = "0.2.0"[dev-dependencies]-criterion = { version = "0.5.1", default-features = false }+criterion = { version = "0.5.1", features = ["plotters", "cargo_bench_support"], default-features = false}rand = "0.8.5"[build-dependencies]diff --git a/harper-core/src/spell/fst_dictionary.rs b/harper-core/src/spell/fst_dictionary.rsindex 6545f56f1..c0b23a3d5 100644--- a/harper-core/src/spell/fst_dictionary.rs+++ b/harper-core/src/spell/fst_dictionary.rs@@ -1,18 +1,15 @@-use fst::Map as FstMap;+use super::FullDictionary;use fst::{automaton::Levenshtein, IntoStreamer};-use itertools::Itertools;+use fst::{Map as FstMap, Streamer};-use super::hunspell::{parse_default_attribute_list, parse_default_word_list};-use crate::{CharString, Lrc, WordMetadata};+use crate::{Lrc, WordMetadata};use super::Dictionary;#[derive(Debug)]pub struct FstDictionary {- /// Sorted by string in lexicographic order- metadata: Vec<WordMetadata>,- /// Sorted in lexicographic order- words: Vec<CharString>,+ /// Underlying FullDictionary used for everything except fuzzy finding+ full_dict: Lrc<FullDictionary>,/// Used for fuzzy-finding the index of words or metadataword_map: FstMap<Vec<u8>>,}@@ -20,27 +17,11 @@ pub struct FstDictionary {/// The uncached function that is used to produce the original copy of the/// curated dictionary.fn uncached_inner_new() -> Lrc<FstDictionary> {- let word_list = parse_default_word_list().unwrap();- let attr_list = parse_default_attribute_list();-- // There will be at _least_ this number of words- // This creates a memory map, which enables searching the map without loading- // all of it into memory.+ let full_dict = FullDictionary::curated();let word_map = FstMap::new(include_bytes!("../../dictionary.fst").to_vec()).unwrap();- // There will be at _least_ this number of words- let mut wmap = hashbrown::HashMap::with_capacity(word_list.len());- attr_list.expand_marked_words(word_list, &mut wmap);-- let (words, metadata) = wmap- .into_iter()- .sorted_by(|a, b| Ord::cmp(a.0.as_slice(), b.0.as_slice()))- .dedup_by(|a, b| a.0 == b.0)- .unzip();-Lrc::new(FstDictionary {- metadata,- words,+ full_dict,word_map,})}@@ -59,29 +40,19 @@ impl FstDictionary {impl Dictionary for FstDictionary {fn contains_word(&self, word: &[char]) -> bool {- self.word_map.contains_key(word.iter().collect::<String>())+ self.full_dict.contains_word(word)}fn contains_word_str(&self, word: &str) -> bool {- self.word_map.contains_key(word)+ self.full_dict.contains_word_str(word)}fn get_word_metadata(&self, word: &[char]) -> WordMetadata {- let index: Option<usize> = self- .word_map- .get(word.iter().collect::<String>())- .map(|i| i as usize);-- if let Some(i) = index {- self.metadata[i]- } else {- WordMetadata::default()- }+ self.full_dict.get_word_metadata(word)}fn get_word_metadata_str(&self, word: &str) -> WordMetadata {- let index: usize = self.word_map.get(word).unwrap() as usize;- self.metadata[index]+ self.full_dict.get_word_metadata_str(word)}fn fuzzy_match(@@ -100,21 +71,36 @@ impl Dictionary for FstDictionary {max_results: usize,) -> Vec<(&[char], u8, WordMetadata)> {let aut = Levenshtein::new(word, max_distance as u32).unwrap();- let word_indexes: Vec<u64> = self.word_map.search(aut).into_stream().into_values();+ let mut word_indexes_stream = self.word_map.search(aut).into_stream();+ let mut word_indexes = Vec::with_capacity(max_results);++ let i = 0;+ while i < max_results {+ if let Some(v) = word_indexes_stream.next() {+ word_indexes.push(v.1);+ } else {+ break;+ }+ }word_indexes.into_iter().take(max_results)- .map(|i| (self.words.get(i as usize).unwrap(), i))- .map(|(word, i)| (word.as_slice(), i as u8, self.metadata[i as usize]))+ .map(|i| (self.full_dict.get_word(i as usize), i))+ .map(|(word, i)| {+ (+ word.as_slice(),+ i as u8,+ self.full_dict.get_metadata(i as usize).to_owned(),+ )+ }).collect()}fn words_iter(&self) -> impl Iterator<Item = &'_ [char]> {- self.words.iter().map(|v| v.as_slice())+ self.full_dict.words_iter()}fn words_with_len_iter(&self, len: usize) -> Box<dyn Iterator<Item = &'_ [char]> + '_> {- // Could be _way_ faster.- Box::new(self.words_iter().filter(move |w| w.len() == len))+ self.full_dict.words_with_len_iter(len)}}diff --git a/harper-core/src/spell/full_dictionary.rs b/harper-core/src/spell/full_dictionary.rsindex 02945ce89..8cd5a85ad 100644--- a/harper-core/src/spell/full_dictionary.rs+++ b/harper-core/src/spell/full_dictionary.rs@@ -1,5 +1,4 @@-use std::borrow::Cow;-+use super::seq_to_normalized;use hashbrown::HashMap;use smallvec::{SmallVec, ToSmallVec};@@ -117,6 +116,13 @@ impl FullDictionary {word_len_starts}++ pub fn get_word(&self, index: usize) -> &CharString {+ &self.words[index]+ }+ pub fn get_metadata(&self, index: usize) -> &WordMetadata {+ self.word_map.get(self.get_word(index)).unwrap()+ }}impl Default for FullDictionary {@@ -291,23 +297,6 @@ pub fn edit_distance(source: &[char], target: &[char]) -> u8 {edit_distance_min_alloc(source, target, &mut Vec::new(), &mut Vec::new())}-/// Convert a given character sequence to the standard character set-/// the dictionary is in.-pub fn seq_to_normalized(seq: &[char]) -> Cow<'_, [char]> {- if seq.iter().any(|c| char_to_normalized(*c) != *c) {- Cow::Owned(seq.iter().copied().map(char_to_normalized).collect())- } else {- Cow::Borrowed(seq)- }-}--pub fn char_to_normalized(c: char) -> char {- match c {- '’' => '\'',- _ => c,- }-}-#[cfg(test)]mod tests {use super::{edit_distance, seq_to_normalized};diff --git a/harper-core/src/spell/mod.rs b/harper-core/src/spell/mod.rsindex f196fdb7f..38b3ecdbb 100644--- a/harper-core/src/spell/mod.rs+++ b/harper-core/src/spell/mod.rs@@ -1,3 +1,5 @@+use std::borrow::Cow;+use itertools::{Itertools, MinMaxResult};use crate::CharString;@@ -84,6 +86,23 @@ pub fn suggest_correct_spelling_str(.collect()}+/// Convert a given character sequence to the standard character set+/// the dictionary is in.+pub fn seq_to_normalized(seq: &[char]) -> Cow<'_, [char]> {+ if seq.iter().any(|c| char_to_normalized(*c) != *c) {+ Cow::Owned(seq.iter().copied().map(char_to_normalized).collect())+ } else {+ Cow::Borrowed(seq)+ }+}++pub fn char_to_normalized(c: char) -> char {+ match c {+ '’' => '\'',+ _ => c,+ }+}+#[cfg(test)]mod tests {use itertools::Itertools;