Offline, privacy-first grammar checker. Fast, open-source, Rust-powered
refactor(core): `MutableDictionary` now uses word hashes for lookup
Elijah Pottere48fb6302142
diff --git a/harper-cli/src/main.rs b/harper-cli/src/main.rsindex 1e9cac48e..1a4a2de2a 100644--- a/harper-cli/src/main.rs+++ b/harper-cli/src/main.rs@@ -1,7 +1,6 @@#![doc = include_str!("../README.md")]use std::collections::BTreeMap;-use std::collections::HashSet;use std::path::{Path, PathBuf};use std::process;@@ -11,13 +10,8 @@ use clap::Parser;use harper_comments::CommentParser;use harper_core::linting::{LintGroup, Linter};use harper_core::parsers::{Markdown, MarkdownOptions};-use harper_core::spell::hunspell::parse_default_attribute_list;-use harper_core::spell::hunspell::parse_default_word_list;-use harper_core::spell::hunspell::word_list::parse_word_list;-use harper_core::spell::hunspell::word_list::MarkedWord;use harper_core::{- remove_overlaps, CharString, CharStringExt, Dictionary, Document, FstDictionary, TokenKind,- TokenStringExt, WordMetadata,+ remove_overlaps, CharStringExt, Dictionary, Document, FstDictionary, TokenKind, TokenStringExt,};use harper_literate_haskell::LiterateHaskellParser;use hashbrown::HashMap;@@ -55,8 +49,6 @@ enum Args {},/// Get the metadata associated with a particular word.Metadata { word: String },- /// Get all the forms of a word using the affixes.- Forms { words: Vec<String> },/// Emit a decompressed, line-separated list of the words in Harper's dictionary.Words,/// Print the default config with descriptions.@@ -202,104 +194,6 @@ fn main() -> anyhow::Result<()> {Ok(())}- Args::Forms { words: args_words } => {- let default_attrs = parse_default_attribute_list();- let args_total = args_words.len();-- let default_marked_words = parse_default_word_list().unwrap();-- for (arg_index, arg_word) in args_words.iter().enumerate() {- let mut arg_bare_word = arg_word.to_string();- let mut arg_annot = None;- if let Some((word_part, annot_part)) = arg_word.split_once('/') {- arg_bare_word = word_part.to_string();- arg_annot = Some(annot_part);- }-- let mut already_in_dict = false;- let mut already_annotated = false;- let old_annot = get_annotations(&default_marked_words, &arg_bare_word);-- if let Some(ref annot) = old_annot {- already_in_dict = true;- already_annotated = !annot.is_empty();- }-- let mut status_summary = match (already_in_dict, already_annotated) {- (true, true) => format!(- "'{}' is already in the dictionary with annotation `{}`.",- arg_bare_word,- old_annot.clone().unwrap()- ),- (true, false) => format!(- "'{}' is already in the dictionary but not annotated.",- arg_bare_word- ),- _ => format!("'{}' is not in the dictionary yet.", arg_bare_word),- };-- let mut different_annot = false;-- if already_annotated && arg_annot.is_some() {- if old_annot- .clone()- .unwrap()- .chars()- .collect::<HashSet<char>>()- != arg_annot.unwrap().chars().collect()- {- different_annot = true;- status_summary- .push_str("\n Your annotations differ from the dictionary.\n");- } else {- status_summary- .push_str("\n Your annotations are the same as the dictionary.\n");- }- }-- println!("{}", status_summary);-- let print_word_info = |arg_index: usize, arg_word: &str| {- let mut our_expanded_word: HashMap<CharString, WordMetadata> = HashMap::new();- let our_marked_word = parse_word_list(&format!("1\n{arg_word}")).unwrap();- default_attrs.expand_marked_words(our_marked_word, &mut our_expanded_word);-- println!(- "{}{}{}",- if arg_index > 0 { "\n" } else { "" },- if args_total != 1 {- format!("{}/{}: ", arg_index + 1, args_total)- } else {- "".to_string()- },- arg_word- );-- our_expanded_word.keys().for_each(|form| {- let string_form: String = form.iter().collect();- println!(" - {}", string_form);- });- };-- let should_print_old_info = already_annotated;- let should_print_new_info =- arg_annot.is_some() && (!already_annotated || different_annot);-- if should_print_old_info {- println!("Old, from the dictionary:");- print_word_info(- arg_index,- &format!("{}/{}", arg_bare_word, old_annot.unwrap()),- );- }- if should_print_new_info {- println!("New, from you:");- print_word_info(arg_index, arg_word);- }- }-- Ok(())- }Args::Config => {#[derive(Serialize)]struct Config {@@ -377,14 +271,3 @@ fn load_file(file: &Path, markdown_options: MarkdownOptions) -> anyhow::Result<(Ok((Document::new_curated(&source, &parser), source))}--fn get_annotations(marked_words: &[MarkedWord], word: &str) -> Option<String> {- for marked_word in marked_words.iter() {- let mw = &marked_word.letters;- let mw = mw.iter().collect::<String>();- if mw == word {- return Some(marked_word.attributes.iter().collect::<String>());- }- }- None-}diff --git a/harper-comments/tests/language_support.rs b/harper-comments/tests/language_support.rsindex fc2d48b88..c0f0c7a40 100644--- a/harper-comments/tests/language_support.rs+++ b/harper-comments/tests/language_support.rs@@ -47,7 +47,7 @@ create_test!(jsdoc.ts, 4);create_test!(issue_96.lua, 0);create_test!(merged_lines.ts, 1);create_test!(javadoc_clean_simple.java, 0);-create_test!(javadoc_complex.java, 4);+create_test!(javadoc_complex.java, 5);create_test!(issue_132.rs, 1);create_test!(laravel_app.php, 2);diff --git a/harper-comments/tests/language_support_sources/javadoc_clean_simple.java b/harper-comments/tests/language_support_sources/javadoc_clean_simple.javaindex 54278f565..74125e621 100644--- a/harper-comments/tests/language_support_sources/javadoc_clean_simple.java+++ b/harper-comments/tests/language_support_sources/javadoc_clean_simple.java@@ -1,7 +1,7 @@class TestClass {/**- * This is a JavaDoc without any of the fancy frills that come with it.+ * This is a Javadoc without any of the fancy frills that come with it.*/public static void main(String[] args) {System.out.println("Hello world.");diff --git a/harper-core/affixes.json b/harper-core/affixes.jsonindex 37f567396..dbd6b0d62 100644--- a/harper-core/affixes.json+++ b/harper-core/affixes.json@@ -672,7 +672,9 @@"replacements": [],"adds_metadata": {},"gifts_metadata": {- "pronoun": {}+ "pronoun": {+ "is_plural": false+ }}},"n": {diff --git a/harper-core/dictionary.dict b/harper-core/dictionary.dictindex 7419cb811..41dabcd27 100644--- a/harper-core/dictionary.dict+++ b/harper-core/dictionary.dict@@ -4644,7 +4644,7 @@ Hymen/2MHyperion/2MHyundai/1MHz/M-I/8~sf+I/8~fI'd/~I'll/~I'm/~@@ -7388,7 +7388,6 @@ OHSA/MOJ/15OK/1452SMDGOMB/2M-ON/2OPEC/2MOR/712OS/251M@@ -26832,7 +26831,7 @@ heppest/5heptagon/1MSheptagonal/5heptathlon/1SM-her/~-8ot+her/~-*otherald/~14SMDGheralded/~4Uheraldic/~5@@ -26929,7 +26928,7 @@ hf/~hgt/1hgwy/1hi/~15D-his/~8-+his/~*-hiatus/~1MShibachi/1MShibernate/4GNDS@@ -34403,7 +34402,6 @@ nympho/1Snymphomania/1Mnymphomaniac/15SMnymphs/~14-o/~15+o'clock/oaf/1SMoafish/5PY@@ -34720,8 +34718,9 @@ omniscience/1Momniscient/51omnivore/1MSomnivorous/~5PY-omnivorousness/1M-on/~5+41Y+omnivorousness/5+on/~5+4+only/~onboard/~54once/~71Moncogene/1SM@@ -48943,7 +48942,7 @@ wayside/15SMwayward/~5PYwaywardness/1Mwazoo/1S-we/~sf:+we/~:we'd/we'll/we're/~@@ -50464,4 +50463,4 @@ Metasploit/21MScryptographic/5Qeasy peasy/5un-American/5-BRICS/\ No newline at end of file+BRICS/diff --git a/harper-core/src/char_string.rs b/harper-core/src/char_string.rsindex ec65ac7fc..74d44efb3 100644--- a/harper-core/src/char_string.rs+++ b/harper-core/src/char_string.rs@@ -9,6 +9,7 @@ pub type CharString = SmallVec<[char; 16]>;/// Extensions to character sequences that make them easier to wrangle.pub trait CharStringExt {fn to_lower(&self) -> Cow<[char]>;+ fn normalized(&self) -> Cow<[char]>;fn to_string(&self) -> String;}@@ -28,6 +29,31 @@ impl CharStringExt for [char] {fn to_string(&self) -> String {self.iter().collect()}++ /// Convert a given character sequence to the standard character set+ /// the dictionary is in.+ fn normalized(&self) -> Cow<[char]> {+ if self.as_ref().iter().any(|c| char_to_normalized(*c) != *c) {+ Cow::Owned(+ self.as_ref()+ .iter()+ .copied()+ .map(char_to_normalized)+ .collect(),+ )+ } else {+ Cow::Borrowed(self)+ }+ }+}++fn char_to_normalized(c: char) -> char {+ match c {+ '’' => '\'',+ '‘' => '\'',+ ''' => '\'',+ _ => c,+ }}macro_rules! char_string {diff --git a/harper-core/src/document.rs b/harper-core/src/document.rsindex 358d3dc37..24db1d0c7 100644--- a/harper-core/src/document.rs+++ b/harper-core/src/document.rs@@ -152,7 +152,7 @@ impl Document {/// When a word that is either an adjective or a noun is sandwiched between an article and a noun,/// it definitely is not a noun.fn articles_imply_nouns(&mut self) {- let pattern = Self::ELLIPSIS_PATTERN.with(|v| v.clone());+ let pattern = Self::ARTICLE_PATTERN.with(|v| v.clone());for m in pattern.find_all_matches_in_doc(self) {if let TokenKind::Word(Some(metadata)) = &mut self.tokens[m.start + 2].kind {diff --git a/harper-core/src/lib.rs b/harper-core/src/lib.rsindex fd39b8b73..11461dc51 100644--- a/harper-core/src/lib.rs+++ b/harper-core/src/lib.rs@@ -38,7 +38,7 @@ pub use mask::{Mask, Masker};pub use number::{Number, NumberSuffix};pub use punctuation::{Punctuation, Quote};pub use span::Span;-pub use spell::{Dictionary, FstDictionary, MergedDictionary, MutableDictionary};+pub use spell::{Dictionary, FstDictionary, MergedDictionary, MutableDictionary, WordId};pub use sync::{LSend, Lrc};pub use title_case::{make_title_case, make_title_case_str};pub use token::Token;diff --git a/harper-core/src/linting/hop_hope/to_hope.rs b/harper-core/src/linting/hop_hope/to_hope.rsindex 6fd1aa5b6..e8353b80f 100644--- a/harper-core/src/linting/hop_hope/to_hope.rs+++ b/harper-core/src/linting/hop_hope/to_hope.rs@@ -10,11 +10,11 @@ pub struct ToHope {impl Default for ToHope {fn default() -> Self {let pattern = SequencePattern::default()- .then_singular_subject()+ .then_not_plural_nominal().then_whitespace().then(WordSet::new(&["hop", "hopped"])).then_whitespace()- .then_singular_subject();+ .then_nominal();Self {pattern: Box::new(pattern),diff --git a/harper-core/src/linting/pique_interest.rs b/harper-core/src/linting/pique_interest.rsindex 8ab62767e..e2c128121 100644--- a/harper-core/src/linting/pique_interest.rs+++ b/harper-core/src/linting/pique_interest.rs@@ -17,7 +17,7 @@ impl Default for PiqueInterest {"peak", "peaked", "peek", "peeked", "peeking", "peaking",])).then_whitespace()- .then_singular_subject()+ .then_not_plural_nominal().then_whitespace().t_aco("interest");diff --git a/harper-core/src/patterns/mod.rs b/harper-core/src/patterns/mod.rsindex db5c3f028..6c60b4f97 100644--- a/harper-core/src/patterns/mod.rs+++ b/harper-core/src/patterns/mod.rs@@ -25,7 +25,6 @@ mod pattern_map;mod repeating_pattern;mod sequence_pattern;mod similar_to_phrase;-mod singular_subject;mod split_compound_word;mod token_kind_pattern_group;mod whitespace_pattern;@@ -50,7 +49,6 @@ pub use pattern_map::PatternMap;pub use repeating_pattern::RepeatingPattern;pub use sequence_pattern::SequencePattern;pub use similar_to_phrase::SimilarToPhrase;-pub use singular_subject::SingularSubject;pub use split_compound_word::SplitCompoundWord;pub use token_kind_pattern_group::TokenKindPatternGroup;pub use whitespace_pattern::WhitespacePattern;diff --git a/harper-core/src/patterns/sequence_pattern.rs b/harper-core/src/patterns/sequence_pattern.rsindex 776ac7d9a..9802147b4 100644--- a/harper-core/src/patterns/sequence_pattern.rs+++ b/harper-core/src/patterns/sequence_pattern.rs@@ -1,9 +1,7 @@use paste::paste;use super::whitespace_pattern::WhitespacePattern;-use super::{- AnyCapitalization, AnyPattern, IndefiniteArticle, Pattern, RepeatingPattern, SingularSubject,-};+use super::{AnyCapitalization, AnyPattern, IndefiniteArticle, Pattern, RepeatingPattern};use crate::{Token, TokenKind};/// A pattern that checks that a sequence of other patterns match.@@ -86,6 +84,7 @@ impl SequencePattern {gen_then_from_is!(determiner);gen_then_from_is!(proper_noun);gen_then_from_is!(preposition);+ gen_then_from_is!(not_plural_nominal);pub fn then_indefinite_article(self) -> Self {self.then(IndefiniteArticle::default())@@ -114,10 +113,6 @@ impl SequencePattern {self}- pub fn then_singular_subject(self) -> Self {- self.then(SingularSubject::default())- }-/// Shorthand for [`Self::any_capitalization_of`].pub fn aco(word: &'static str) -> Self {Self::any_capitalization_of(word)diff --git a/harper-core/src/patterns/singular_subject.rs b/harper-core/src/patterns/singular_subject.rsdeleted file mode 100644index 4adfe1faa..000000000--- a/harper-core/src/patterns/singular_subject.rs+++ /dev/null@@ -1,23 +0,0 @@-use crate::Token;--use super::{EitherPattern, OwnedPatternExt, Pattern, SequencePattern};--pub struct SingularSubject {- inner: EitherPattern,-}--impl Default for SingularSubject {- fn default() -> Self {- Self {- inner: SequencePattern::default()- .then_pronoun()- .or(Box::new(SequencePattern::default().then_proper_noun())),- }- }-}--impl Pattern for SingularSubject {- fn matches(&self, tokens: &[Token], source: &[char]) -> usize {- self.inner.matches(tokens, source)- }-}diff --git a/harper-core/src/spell/dictionary.rs b/harper-core/src/spell/dictionary.rsindex 42fb11ade..d3cde48b6 100644--- a/harper-core/src/spell/dictionary.rs+++ b/harper-core/src/spell/dictionary.rs@@ -1,6 +1,7 @@use blanket::blanket;use super::FuzzyMatchResult;+use super::WordId;use crate::WordMetadata;/// An in-memory database that contains everything necessary to parse and analyze English text.@@ -37,12 +38,13 @@ pub trait Dictionary: Send + Sync {/// If the word isn't in the dictionary, the resulting metadata will be/// empty.fn get_word_metadata_str(&self, word: &str) -> Option<&WordMetadata>;+/// Iterate over the words in the dictionary.fn words_iter(&self) -> Box<dyn Iterator<Item = &'_ [char]> + Send + '_>;/// The number of words in the dictionary.fn word_count(&self) -> usize;- /// Iterate over all the words in the dictionary of a given length- fn words_with_len_iter(&self, len: usize) -> Box<dyn Iterator<Item = &'_ [char]> + Send + '_>;+ /// Returns the correct capitalization of the word with the given ID.+ fn get_word_from_id(&self, id: &WordId) -> Option<&[char]>;}diff --git a/harper-core/src/spell/fst_dictionary.rs b/harper-core/src/spell/fst_dictionary.rsindex e637eeb3c..f7ddf8013 100644--- a/harper-core/src/spell/fst_dictionary.rs+++ b/harper-core/src/spell/fst_dictionary.rs@@ -1,10 +1,9 @@use super::{- MutableDictionary,+ MutableDictionary, WordId,hunspell::{parse_default_attribute_list, parse_default_word_list},- seq_to_normalized,+ word_map::WordMap,};use fst::{IntoStreamer, Map as FstMap, Streamer, map::StreamWithState};-use hashbrown::HashMap;use lazy_static::lazy_static;use levenshtein_automata::{DFA, LevenshteinAutomatonBuilder};use std::{cell::RefCell, sync::Arc};@@ -34,10 +33,15 @@ fn uncached_inner_new() -> Arc<FstDictionary> {let attr_list = parse_default_attribute_list();// There will be at _least_ this number of words- let mut word_map = HashMap::with_capacity(word_list.len());+ let mut word_map = WordMap::with_capacity(word_list.len());attr_list.expand_marked_words(word_list, &mut word_map);- Arc::new(FstDictionary::new(word_map))+ Arc::new(FstDictionary::new(+ word_map+ .into_iter()+ .map(|entry| (entry.canonical_spelling, entry.metadata))+ .collect(),+ ))}const EXPECTED_DISTANCE: u8 = 3;@@ -71,8 +75,7 @@ impl FstDictionary {(*DICT).clone()}- pub fn new(new_words: HashMap<CharString, WordMetadata>) -> Self {- let mut words: Vec<(CharString, WordMetadata)> = new_words.into_iter().collect();+ pub fn new(mut words: Vec<(CharString, WordMetadata)>) -> Self {words.sort_unstable_by(|(a, _), (b, _)| a.cmp(b));words.dedup_by(|(a, _), (b, _)| a == b);@@ -151,7 +154,7 @@ impl Dictionary for FstDictionary {max_distance: u8,max_results: usize,) -> Vec<FuzzyMatchResult> {- let misspelled_word_charslice = seq_to_normalized(word);+ let misspelled_word_charslice = word.normalized();let misspelled_word_string = misspelled_word_charslice.to_string();// Actual FST search@@ -210,10 +213,6 @@ impl Dictionary for FstDictionary {self.full_dict.words_iter()}- fn words_with_len_iter(&self, len: usize) -> Box<dyn Iterator<Item = &'_ [char]> + Send + '_> {- self.full_dict.words_with_len_iter(len)- }-fn word_count(&self) -> usize {self.full_dict.word_count()}@@ -229,6 +228,10 @@ impl Dictionary for FstDictionary {fn get_correct_capitalization_of(&self, word: &[char]) -> Option<&'_ [char]> {self.full_dict.get_correct_capitalization_of(word)}++ fn get_word_from_id(&self, id: &WordId) -> Option<&[char]> {+ self.full_dict.get_word_from_id(id)+ }}#[cfg(test)]@@ -236,7 +239,7 @@ mod tests {use itertools::Itertools;use crate::CharStringExt;- use crate::{Dictionary, spell::seq_to_normalized};+ use crate::Dictionary;use super::FstDictionary;@@ -245,7 +248,7 @@ mod tests {let dict = FstDictionary::curated();for word in dict.words_iter() {- let misspelled_normalized = seq_to_normalized(word);+ let misspelled_normalized = word.normalized();let misspelled_word = misspelled_normalized.to_string();let misspelled_lower = misspelled_normalized.to_lower().to_string();@@ -264,9 +267,9 @@ mod tests {let dict = FstDictionary::curated();let word: Vec<_> = "hello".chars().collect();- let misspelled_normalized = seq_to_normalized(&word);- let misspelled_word: String = misspelled_normalized.to_string();- let misspelled_lower: String = misspelled_normalized.to_lower().to_string();+ let misspelled_normalized = word.normalized();+ let misspelled_word = misspelled_normalized.to_string();+ let misspelled_lower = misspelled_normalized.to_lower().to_string();assert!(dict.contains_word(&misspelled_normalized));assert!(@@ -275,6 +278,13 @@ mod tests {);}+ #[test]+ fn on_is_not_nominal() {+ let dict = FstDictionary::curated();++ assert!(!dict.get_word_metadata_str("on").unwrap().is_nominal());+ }+#[test]fn fuzzy_result_sorted_by_edit_distance() {let dict = FstDictionary::curated();diff --git a/harper-core/src/spell/hunspell/attribute_list.rs b/harper-core/src/spell/hunspell/attribute_list.rsindex d84f2ffc8..ef637cbda 100644--- a/harper-core/src/spell/hunspell/attribute_list.rs+++ b/harper-core/src/spell/hunspell/attribute_list.rs@@ -2,6 +2,7 @@ use hashbrown::HashMap;use serde::{Deserialize, Serialize};use smallvec::ToSmallVec;+use super::super::word_map::{WordMap, WordMapEntry};use super::Error;use super::affix_replacement::AffixReplacement;use super::expansion::{Expansion, HumanReadableExpansion};@@ -30,11 +31,7 @@ impl AttributeList {/// Will append to the given `dest`;////// In the future, I want to make this function cleaner and faster.- pub fn expand_marked_word(- &self,- word: MarkedWord,- dest: &mut HashMap<CharString, WordMetadata>,- ) {+ pub fn expand_marked_word(&self, word: MarkedWord, dest: &mut WordMap) {dest.reserve(word.attributes.len() + 1);let mut gifted_metadata = WordMetadata::default();@@ -78,24 +75,33 @@ impl AttributeList {},dest,);- let t_metadata = dest.get_mut(&new_word).unwrap();+ let t_metadata = dest.get_metadata_mut_chars(&new_word).unwrap();t_metadata.append(&metadata);}} else {for (key, value) in new_words.into_iter() {- if let Some(val) = dest.get_mut(&key) {+ if let Some(val) = dest.get_metadata_mut_chars(&key) {val.append(&value);} else {- dest.insert(key, value);+ dest.insert(WordMapEntry {+ canonical_spelling: key,+ metadata: value,+ });}}}}- if let Some(prev_val) = dest.get(&word.letters) {- dest.insert(word.letters, gifted_metadata.or(prev_val));+ if let Some(prev_val) = dest.get_with_chars(&word.letters) {+ dest.insert(WordMapEntry {+ metadata: gifted_metadata.or(&prev_val.metadata),+ canonical_spelling: word.letters,+ });} else {- dest.insert(word.letters, gifted_metadata);+ dest.insert(WordMapEntry {+ metadata: gifted_metadata,+ canonical_spelling: word.letters,+ });}}@@ -105,7 +111,7 @@ impl AttributeList {pub fn expand_marked_words(&self,words: impl IntoIterator<Item = MarkedWord>,- dest: &mut HashMap<CharString, WordMetadata>,+ dest: &mut WordMap,) {for word in words {self.expand_marked_word(word, dest);diff --git a/harper-core/src/spell/hunspell/mod.rs b/harper-core/src/spell/hunspell/mod.rsindex 22e038305..518ce498f 100644--- a/harper-core/src/spell/hunspell/mod.rs+++ b/harper-core/src/spell/hunspell/mod.rs@@ -28,13 +28,14 @@ pub fn parse_default_attribute_list() -> AttributeList {#[cfg(test)]mod tests {- use hashbrown::{HashMap, HashSet};+ use hashbrown::HashSet;use serde_json::json;+ use super::super::word_map::WordMap;use super::word_list::parse_word_list;use super::{parse_default_attribute_list, parse_default_word_list};+ use crate::CharStringExt;use crate::spell::hunspell::attribute_list::HumanReadableAttributeList;- use crate::{CharString, WordMetadata};pub const TEST_WORD_LIST: &str = "3\nhello\ntry/B\nwork/AB";@@ -85,12 +86,12 @@ mod tests {.unwrap();let attributes = attributes.into_normal().unwrap();- let mut expanded = HashMap::new();+ let mut expanded = WordMap::default();attributes.expand_marked_words(words, &mut expanded);let expanded: HashSet<String> = expanded.into_iter()- .map(|v| v.0.into_iter().collect())+ .map(|v| v.canonical_spelling.to_string()).collect();assert_eq!(@@ -157,22 +158,22 @@ mod tests {.unwrap();let attributes = attributes.into_normal().unwrap();- let mut expanded: HashMap<CharString, WordMetadata> = HashMap::new();+ let mut expanded = WordMap::default();attributes.expand_marked_words(words, &mut expanded);- let giant_data = expanded.get(&split("giant")).unwrap();- assert!(giant_data.is_noun());+ let giant_data = expanded.get_with_str("giant").unwrap();+ assert!(giant_data.metadata.is_noun());- let giants_data = expanded.get(&split("giants")).unwrap();- assert!(giants_data.is_plural_noun());+ let giants_data = expanded.get_with_str("giants").unwrap();+ assert!(giants_data.metadata.is_plural_noun());}- fn build_expanded() -> HashMap<CharString, WordMetadata> {+ fn build_expanded() -> WordMap {let words = parse_default_word_list().unwrap();let attributes = parse_default_attribute_list();- let mut expanded = HashMap::new();+ let mut expanded = WordMap::default();attributes.expand_marked_words(words, &mut expanded);@@ -186,29 +187,30 @@ mod tests {#[test]fn expanded_contains_giants() {- assert!(build_expanded().contains_key(&split("giants")));+ assert!(build_expanded().contains_str("giants"));}#[test]fn expanded_contains_deallocate() {- assert!(build_expanded().contains_key(&split("deallocate")));+ assert!(build_expanded().contains_str("deallocate"));}#[test]fn expanded_contains_repo() {let expanded = build_expanded();- assert!(expanded.contains_key(&split("repo")));- assert!(expanded.contains_key(&split("repos")));- assert!(expanded.contains_key(&split("repo's")));+ assert!(expanded.contains_str("repo"));+ assert!(expanded.contains_str("repos"));+ assert!(expanded.contains_str("repo's"));}#[test]fn expanded_contains_possessive_abandonment() {assert!(build_expanded()- .get(&split("abandonment's"))+ .get_with_str("abandonment's").unwrap()+ .metadata.is_possessive_noun())}@@ -217,21 +219,21 @@ mod tests {fn has_is_not_a_nominal() {let expanded = build_expanded();- let has = expanded.get(&split("has"));+ let has = expanded.get_with_str("has");assert!(has.is_some());- assert!(!has.unwrap().is_nominal(),)+ assert!(!has.unwrap().metadata.is_nominal())}#[test]fn is_is_linking_verb() {let expanded = build_expanded();- let is = expanded.get(&split("is"));+ let is = expanded.get_with_str("is");dbg!(&is);assert!(is.is_some());- assert!(is.unwrap().is_linking_verb());+ assert!(is.unwrap().metadata.is_linking_verb());}#[test]@@ -242,19 +244,19 @@ mod tests {let merged_attrs = parse_default_attribute_list();let spread_attrs = parse_default_attribute_list();- let mut expanded1 = HashMap::new();- let mut expanded2 = HashMap::new();+ let mut expanded1 = WordMap::default();+ let mut expanded2 = WordMap::default();merged_attrs.expand_marked_words(merged_word, &mut expanded1);let expanded_merged: HashSet<String> = expanded1.into_iter()- .map(|v| v.0.into_iter().collect())+ .map(|v| v.canonical_spelling.into_iter().collect()).collect();spread_attrs.expand_marked_words(spread_word, &mut expanded2);let expanded_spread: HashSet<String> = expanded2.into_iter()- .map(|v| v.0.into_iter().collect())+ .map(|v| v.canonical_spelling.into_iter().collect()).collect();assert_eq!(@@ -262,8 +264,4 @@ mod tests {expanded_spread.into_iter().collect::<HashSet<_>>());}-- fn split(text: &str) -> CharString {- text.chars().collect()- }}diff --git a/harper-core/src/spell/merged_dictionary.rs b/harper-core/src/spell/merged_dictionary.rsindex ce04ac54c..5792db00a 100644--- a/harper-core/src/spell/merged_dictionary.rs+++ b/harper-core/src/spell/merged_dictionary.rs@@ -4,7 +4,7 @@ use std::sync::Arc;use foldhash::quality::FixedState;use itertools::Itertools;-use super::FstDictionary;+use super::{FstDictionary, WordId};use super::{FuzzyMatchResult, dictionary::Dictionary};use crate::{CharString, WordMetadata};@@ -107,14 +107,6 @@ impl Dictionary for MergedDictionary {Box::new(self.children.iter().flat_map(|c| c.words_iter()))}- fn words_with_len_iter(&self, len: usize) -> Box<dyn Iterator<Item = &'_ [char]> + Send + '_> {- Box::new(- self.children- .iter()- .flat_map(move |c| c.words_with_len_iter(len)),- )- }-fn contains_word_str(&self, word: &str) -> bool {let chars: CharString = word.chars().collect();self.contains_word(&chars)@@ -161,4 +153,10 @@ impl Dictionary for MergedDictionary {fn word_count(&self) -> usize {self.children.iter().map(|d| d.word_count()).sum()}++ fn get_word_from_id(&self, id: &WordId) -> Option<&[char]> {+ self.children+ .iter()+ .find_map(|dict| dict.get_word_from_id(id))+ }}diff --git a/harper-core/src/spell/mod.rs b/harper-core/src/spell/mod.rsindex c53364c22..1db80ef74 100644--- a/harper-core/src/spell/mod.rs+++ b/harper-core/src/spell/mod.rs@@ -1,17 +1,18 @@-use std::borrow::Cow;-use crate::{CharString, CharStringExt, WordMetadata};pub use self::dictionary::Dictionary;pub use self::fst_dictionary::FstDictionary;pub use self::merged_dictionary::MergedDictionary;pub use self::mutable_dictionary::MutableDictionary;+pub use self::word_id::WordId;mod dictionary;mod fst_dictionary;pub mod hunspell;mod merged_dictionary;mod mutable_dictionary;+mod word_id;+mod word_map;#[derive(PartialEq, Debug, Hash, Eq)]pub struct FuzzyMatchResult<'a> {@@ -95,30 +96,13 @@ pub fn suggest_correct_spelling_str(.collect()}-/// Convert a given character sequence to the standard character set-/// the dictionary is in.-fn seq_to_normalized(seq: &[char]) -> Cow<'_, [char]> {- if seq.iter().any(|c| char_to_normalized(*c) != *c) {- Cow::Owned(seq.iter().copied().map(char_to_normalized).collect())- } else {- Cow::Borrowed(seq)- }-}--fn char_to_normalized(c: char) -> char {- match c {- '’' => '\'',- '‘' => '\'',- ''' => '\'',- _ => c,- }-}-#[cfg(test)]mod tests {use itertools::Itertools;- use super::{FstDictionary, seq_to_normalized, suggest_correct_spelling_str};+ use crate::CharStringExt;++ use super::{FstDictionary, suggest_correct_spelling_str};const RESULT_LIMIT: usize = 100;const MAX_EDIT_DIST: u8 = 2;@@ -126,7 +110,7 @@ mod tests {#[test]fn normalizes_weve() {let word = vec!['w', 'e', '’', 'v', 'e'];- let norm = seq_to_normalized(&word);+ let norm = word.normalized();assert_eq!(norm.clone(), vec!['w', 'e', '\'', 'v', 'e'])}diff --git a/harper-core/src/spell/mutable_dictionary.rs b/harper-core/src/spell/mutable_dictionary.rsindex 5a35f1e09..15f46dd7b 100644--- a/harper-core/src/spell/mutable_dictionary.rs+++ b/harper-core/src/spell/mutable_dictionary.rs@@ -1,12 +1,11 @@use super::{+ WordId,hunspell::{parse_default_attribute_list, parse_default_word_list},- seq_to_normalized,+ word_map::{WordMap, WordMapEntry},};use crate::edit_distance::edit_distance_min_alloc;-use hashbrown::HashMap;use itertools::Itertools;use lazy_static::lazy_static;-use smallvec::{SmallVec, ToSmallVec};use std::sync::Arc;use crate::{CharString, CharStringExt, WordMetadata};@@ -24,25 +23,8 @@ use super::dictionary::Dictionary;/// [`super::MergedDictionary`].#[derive(Debug, Clone, Eq, PartialEq)]pub struct MutableDictionary {- /// Storing a separate [`Vec`] for iterations speeds up spellchecking by- /// ~16% at the cost of additional memory.- ///- /// This is likely due to increased locality 🤷.- ///- /// This list is sorted by word length (i.e. the shortest words are first).- words: Vec<CharString>,- /// A lookup list for each word length.- /// Each index of this list will return the first index of [`Self::words`]- /// that has a word whose index is that length.- word_len_starts: Vec<usize>,/// All English words- word_map: HashMap<CharString, WordMetadata>,- /// A map from the lowercase versions of a word to the correct capitalization- /// of that same word.- ///- /// It can be used to check if a word is correctly capitalized, or if it is valid, regardless of- /// capitalization.- word_map_lowercase: HashMap<CharString, CharString>,+ word_map: WordMap,}/// The uncached function that is used to produce the original copy of the@@ -52,27 +34,11 @@ fn uncached_inner_new() -> Arc<MutableDictionary> {let attr_list = parse_default_attribute_list();// There will be at _least_ this number of words- let mut word_map = HashMap::with_capacity(word_list.len());+ let mut word_map = WordMap::default();attr_list.expand_marked_words(word_list, &mut word_map);- let mut words: Vec<CharString> = word_map.iter().map(|(v, _)| v.clone()).collect();-- words.sort_unstable();- words.dedup();- words.sort_unstable_by_key(|w| w.len());-- let mut word_map_lowercase = HashMap::with_capacity(word_map.len());- for key in word_map.keys() {- word_map_lowercase.insert(key.to_lower().to_smallvec(), key.clone());- }-- Arc::new(MutableDictionary {- word_map,- word_map_lowercase,- word_len_starts: MutableDictionary::create_len_starts(&words),- words,- })+ Arc::new(MutableDictionary { word_map })}lazy_static! {@@ -82,10 +48,7 @@ lazy_static! {impl MutableDictionary {pub fn new() -> Self {Self {- words: Vec::new(),- word_len_starts: Vec::new(),- word_map: HashMap::new(),- word_map_lowercase: HashMap::new(),+ word_map: WordMap::default(),}}@@ -103,22 +66,12 @@ impl MutableDictionary {&mut self,words: impl IntoIterator<Item = (impl AsRef<[char]>, WordMetadata)>,) {- let pairs: Vec<_> = words- .into_iter()- .filter_map(|(v, m)| {- (!self.contains_word(v.as_ref())).then(|| (v.as_ref().to_smallvec(), m))+ for (chars, metadata) in words.into_iter() {+ self.word_map.insert(WordMapEntry {+ metadata,+ canonical_spelling: chars.as_ref().into(),})- .collect();-- self.words.extend(pairs.iter().map(|(v, _)| v.clone()));- self.words.sort_by_key(|w| w.len());- self.word_len_starts = Self::create_len_starts(&self.words);- self.word_map_lowercase.extend(- pairs- .iter()- .map(|(key, _)| (key.to_lower().to_smallvec(), key.clone())),- );- self.word_map.extend(pairs);+ }}/// Append a single word to the dictionary.@@ -136,24 +89,6 @@ impl MutableDictionary {pub fn append_word_str(&mut self, word: &str, metadata: WordMetadata) {self.append_word(word.chars().collect::<Vec<_>>(), metadata)}-- /// Create a lookup table for finding words of a specific length in a word- /// list.- fn create_len_starts(words: &[CharString]) -> Vec<usize> {- let mut len_words: Vec<_> = words.to_vec();- len_words.sort_by_key(|a| a.len());-- let mut word_len_starts = vec![0, 0];-- for (index, len) in len_words.iter().map(SmallVec::len).enumerate() {- if word_len_starts.len() <= len {- word_len_starts.resize(len, index);- word_len_starts.push(index);- }- }-- word_len_starts- }}impl Default for MutableDictionary {@@ -164,17 +99,11 @@ impl Default for MutableDictionary {impl Dictionary for MutableDictionary {fn get_word_metadata(&self, word: &[char]) -> Option<&WordMetadata> {- let normalized = seq_to_normalized(word);- let correct_caps = self.get_correct_capitalization_of(&normalized)?;-- self.word_map.get(correct_caps)+ self.word_map.get_with_chars(word).map(|v| &v.metadata)}fn contains_word(&self, word: &[char]) -> bool {- let normalized = seq_to_normalized(word);- let lowercase = normalized.to_lower();-- self.word_map_lowercase.contains_key(lowercase.as_ref())+ self.word_map.contains_chars(word)}fn contains_word_str(&self, word: &str) -> bool {@@ -188,12 +117,9 @@ impl Dictionary for MutableDictionary {}fn get_correct_capitalization_of(&self, word: &[char]) -> Option<&'_ [char]> {- let normalized = seq_to_normalized(word);- let lowercase = normalized.to_lower();-- self.word_map_lowercase- .get(lowercase.as_ref())- .map(|v| v.as_slice())+ self.word_map+ .get_with_chars(word)+ .map(|v| v.canonical_spelling.as_slice())}/// Suggest a correct spelling for a given misspelled word.@@ -206,7 +132,7 @@ impl Dictionary for MutableDictionary {max_distance: u8,max_results: usize,) -> Vec<FuzzyMatchResult> {- let misspelled_charslice = seq_to_normalized(word);+ let misspelled_charslice = word.normalized();let misspelled_charslice_lower = misspelled_charslice.to_lower();let shortest_word_len = if misspelled_charslice.len() <= max_distance as usize {@@ -217,9 +143,9 @@ impl Dictionary for MutableDictionary {let longest_word_len = misspelled_charslice.len() + max_distance as usize;// Get candidate words- let words_to_search = (shortest_word_len..=longest_word_len)- .rev()- .flat_map(|len| self.words_with_len_iter(len));+ let words_to_search = self+ .words_iter()+ .filter(|word| (shortest_word_len..=longest_word_len).contains(&word.len()));// Pre-allocated vectors for the edit-distance calculation// 53 is the length of the longest word.@@ -266,36 +192,37 @@ impl Dictionary for MutableDictionary {}fn words_iter(&self) -> Box<dyn Iterator<Item = &'_ [char]> + Send + '_> {- Box::new(self.words.iter().map(|v| v.as_slice()))- }-- fn words_with_len_iter(&self, len: usize) -> Box<dyn Iterator<Item = &'_ [char]> + Send + '_> {- if len == 0 || len >= self.word_len_starts.len() {- return Box::new(std::iter::empty());- }-- let start = self.word_len_starts[len];- let end = if len + 1 == self.word_len_starts.len() {- self.words.len()- } else {- self.word_len_starts[len + 1]- };-- Box::new(self.words[start..end].iter().map(|v| v.as_slice()))+ Box::new(+ self.word_map+ .iter()+ .map(|v| v.canonical_spelling.as_slice()),+ )}fn word_count(&self) -> usize {- self.words.len()+ self.word_map.len()}fn contains_exact_word(&self, word: &[char]) -> bool {- self.word_map.contains_key(seq_to_normalized(word).as_ref())+ let normalized = word.normalized();++ if let Some(found) = self.word_map.get_with_chars(normalized.as_ref()) {+ if found.canonical_spelling.as_ref() == normalized.as_ref() {+ return true;+ }+ }++ false}fn contains_exact_word_str(&self, word: &str) -> bool {let word: CharString = word.chars().collect();self.contains_exact_word(word.as_ref())}++ fn get_word_from_id(&self, id: &WordId) -> Option<&[char]> {+ self.word_map.get(id).map(|w| w.canonical_spelling.as_ref())+ }}#[cfg(test)]@@ -305,19 +232,10 @@ mod tests {use crate::{Dictionary, MutableDictionary};- #[test]- fn words_with_len_contains_self() {- let dict = MutableDictionary::curated();-- let word: CharString = "hello".chars().collect();- let words_with_same_len = dict.words_with_len_iter(word.len()).collect_vec();- assert!(words_with_same_len.contains(&&word[..]));- }-#[test]fn curated_contains_no_duplicates() {let dict = MutableDictionary::curated();- assert!(dict.words.iter().all_unique());+ assert!(dict.words_iter().all_unique());}#[test]diff --git a/harper-core/src/spell/word_id.rs b/harper-core/src/spell/word_id.rsnew file mode 100644index 000000000..d889f9c3e--- /dev/null+++ b/harper-core/src/spell/word_id.rs@@ -0,0 +1,26 @@+use std::hash::BuildHasher;++use foldhash::fast::FixedState;++use crate::CharStringExt;++/// An identifier for a particular word.+///+/// It works by hashing the word it represents, normalized to lowercase.+/// It is meant for situations where you need to refer to a word (or a collection of words),+/// without storing all of accompanying data (like spelling or metadata).+#[derive(Hash, Copy, Clone, PartialEq, Eq, PartialOrd, Debug)]+pub struct WordId {+ hash: u64,+}++impl WordId {+ /// Create a Word ID from+ pub fn from_word_chars(chars: impl AsRef<[char]>) -> Self {+ let normalized = chars.as_ref().normalized();+ let lower = normalized.to_lower();+ let hash = FixedState::default().hash_one(lower);++ Self { hash }+ }+}diff --git a/harper-core/src/spell/word_map.rs b/harper-core/src/spell/word_map.rsnew file mode 100644index 000000000..66a40f2f3--- /dev/null+++ b/harper-core/src/spell/word_map.rs@@ -0,0 +1,104 @@+use hashbrown::{HashMap, hash_map::IntoValues};++use crate::{CharString, WordMetadata};++use super::WordId;++/// The underlying data structure for the `MutableDictionary`.+#[derive(Debug, Clone, Eq, PartialEq, Default)]+pub struct WordMap {+ inner: HashMap<WordId, WordMapEntry>,+}++#[derive(Debug, Clone, Eq, PartialEq)]+pub struct WordMapEntry {+ pub metadata: WordMetadata,+ pub canonical_spelling: CharString,+}++impl WordMap {+ /// Get an entry from the word map using raw chars.+ pub fn get_with_str(&self, string: &str) -> Option<&WordMapEntry> {+ let chars: CharString = string.chars().collect();+ let id = WordId::from_word_chars(chars);++ self.get(&id)+ }++ pub fn contains_str(&self, string: &str) -> bool {+ self.get_with_str(string).is_some()+ }++ pub fn contains_chars(&self, chars: impl AsRef<[char]>) -> bool {+ self.get_with_chars(chars).is_some()+ }++ pub fn contains(&self, id: &WordId) -> bool {+ self.get(id).is_some()+ }++ /// Get an entry from the word map using raw chars.+ pub fn get_with_chars(&self, chars: impl AsRef<[char]>) -> Option<&WordMapEntry> {+ let id = WordId::from_word_chars(chars);++ self.get(&id)+ }++ /// Get an entry from the word map using a word identifier.+ pub fn get(&self, id: &WordId) -> Option<&WordMapEntry> {+ self.inner.get(id)+ }++ /// Borrow a word's metadata mutably+ pub fn get_metadata_mut_chars(+ &mut self,+ chars: impl AsRef<[char]>,+ ) -> Option<&mut WordMetadata> {+ let id = WordId::from_word_chars(chars);++ self.get_metadata_mut(&id)+ }++ /// Borrow a word's metadata mutably+ pub fn get_metadata_mut(&mut self, id: &WordId) -> Option<&mut WordMetadata> {+ self.inner.get_mut(id).map(|v| &mut v.metadata)+ }++ pub fn insert(&mut self, entry: WordMapEntry) {+ let id = WordId::from_word_chars(&entry.canonical_spelling);++ self.inner.insert(id, entry);+ }++ /// Reserves capacity for at least `additional` more elements to be inserted+ /// in the `WordMap`. The collection may reserve more space to avoid+ /// frequent reallocations.+ pub fn reserve(&mut self, additional: usize) {+ self.inner.reserve(additional);+ }++ /// Iterate through the canonical spellings of the words in the map.+ pub fn iter(&self) -> impl Iterator<Item = &WordMapEntry> {+ self.inner.values()+ }++ pub fn len(&self) -> usize {+ self.inner.len()+ }++ pub fn with_capacity(capacity: usize) -> Self {+ Self {+ inner: HashMap::with_capacity(capacity),+ }+ }+}++impl IntoIterator for WordMap {+ type Item = WordMapEntry;++ fn into_iter(self) -> Self::IntoIter {+ self.inner.into_values()+ }++ type IntoIter = IntoValues<WordId, WordMapEntry>;+}diff --git a/harper-core/src/word_metadata.rs b/harper-core/src/word_metadata.rsindex 3bc1f9862..74bfc06dc 100644--- a/harper-core/src/word_metadata.rs+++ b/harper-core/src/word_metadata.rs@@ -2,6 +2,8 @@ use is_macro::Is;use paste::paste;use serde::{Deserialize, Serialize};+use crate::WordId;+#[derive(Debug, Default, Clone, PartialEq, Eq, Serialize, Deserialize, PartialOrd, Hash)]pub struct WordMetadata {pub noun: Option<NounData>,@@ -24,6 +26,8 @@ pub struct WordMetadata {/// Whether the word is considered especially common.#[serde(default = "default_false")]pub common: bool,+ #[serde(skip_serializing, skip_deserializing)]+ pub derived_from: Option<WordId>,}/// Needed for `serde`@@ -105,6 +109,7 @@ impl WordMetadata {determiner: self.determiner || other.determiner,preposition: self.preposition || other.preposition,common: self.common || other.common,+ derived_from: self.derived_from.or(other.derived_from),}}@@ -164,7 +169,7 @@ impl WordMetadata {is_plural: Some(false),..})- ) && matches!(+ ) || matches!(self.pronoun,Some(PronounData {is_plural: Some(false),@@ -195,7 +200,7 @@ impl WordMetadata {matches!(self.swear, Some(true))}- /// Same thing as [`Self::or`], except in-place rather than a copy.+ /// Same thing as [`Self::or`], except in-place rather than a clone.pub fn append(&mut self, other: &Self) -> &mut Self {*self = self.or(other);self