Skip to content

Offline, privacy-first grammar checker. Fast, open-source, Rust-powered

refactor(core): `MutableDictionary` now uses word hashes for lookup

Elijah Pottere48fb6302142
diff --git a/harper-cli/src/main.rs b/harper-cli/src/main.rs
index 1e9cac48e..1a4a2de2a 100644
--- a/harper-cli/src/main.rs
+++ b/harper-cli/src/main.rs
@@ -1,7 +1,6 @@
#![doc = include_str!("../README.md")]
use std::collections::BTreeMap;
-use std::collections::HashSet;
use std::path::{Path, PathBuf};
use std::process;
@@ -11,13 +10,8 @@ use clap::Parser;
use harper_comments::CommentParser;
use harper_core::linting::{LintGroup, Linter};
use harper_core::parsers::{Markdown, MarkdownOptions};
-use harper_core::spell::hunspell::parse_default_attribute_list;
-use harper_core::spell::hunspell::parse_default_word_list;
-use harper_core::spell::hunspell::word_list::parse_word_list;
-use harper_core::spell::hunspell::word_list::MarkedWord;
use harper_core::{
- remove_overlaps, CharString, CharStringExt, Dictionary, Document, FstDictionary, TokenKind,
- TokenStringExt, WordMetadata,
+ remove_overlaps, CharStringExt, Dictionary, Document, FstDictionary, TokenKind, TokenStringExt,
};
use harper_literate_haskell::LiterateHaskellParser;
use hashbrown::HashMap;
@@ -55,8 +49,6 @@ enum Args {
},
/// Get the metadata associated with a particular word.
Metadata { word: String },
- /// Get all the forms of a word using the affixes.
- Forms { words: Vec<String> },
/// Emit a decompressed, line-separated list of the words in Harper's dictionary.
Words,
/// Print the default config with descriptions.
@@ -202,104 +194,6 @@ fn main() -> anyhow::Result<()> {
Ok(())
}
- Args::Forms { words: args_words } => {
- let default_attrs = parse_default_attribute_list();
- let args_total = args_words.len();
-
- let default_marked_words = parse_default_word_list().unwrap();
-
- for (arg_index, arg_word) in args_words.iter().enumerate() {
- let mut arg_bare_word = arg_word.to_string();
- let mut arg_annot = None;
- if let Some((word_part, annot_part)) = arg_word.split_once('/') {
- arg_bare_word = word_part.to_string();
- arg_annot = Some(annot_part);
- }
-
- let mut already_in_dict = false;
- let mut already_annotated = false;
- let old_annot = get_annotations(&default_marked_words, &arg_bare_word);
-
- if let Some(ref annot) = old_annot {
- already_in_dict = true;
- already_annotated = !annot.is_empty();
- }
-
- let mut status_summary = match (already_in_dict, already_annotated) {
- (true, true) => format!(
- "'{}' is already in the dictionary with annotation `{}`.",
- arg_bare_word,
- old_annot.clone().unwrap()
- ),
- (true, false) => format!(
- "'{}' is already in the dictionary but not annotated.",
- arg_bare_word
- ),
- _ => format!("'{}' is not in the dictionary yet.", arg_bare_word),
- };
-
- let mut different_annot = false;
-
- if already_annotated && arg_annot.is_some() {
- if old_annot
- .clone()
- .unwrap()
- .chars()
- .collect::<HashSet<char>>()
- != arg_annot.unwrap().chars().collect()
- {
- different_annot = true;
- status_summary
- .push_str("\n Your annotations differ from the dictionary.\n");
- } else {
- status_summary
- .push_str("\n Your annotations are the same as the dictionary.\n");
- }
- }
-
- println!("{}", status_summary);
-
- let print_word_info = |arg_index: usize, arg_word: &str| {
- let mut our_expanded_word: HashMap<CharString, WordMetadata> = HashMap::new();
- let our_marked_word = parse_word_list(&format!("1\n{arg_word}")).unwrap();
- default_attrs.expand_marked_words(our_marked_word, &mut our_expanded_word);
-
- println!(
- "{}{}{}",
- if arg_index > 0 { "\n" } else { "" },
- if args_total != 1 {
- format!("{}/{}: ", arg_index + 1, args_total)
- } else {
- "".to_string()
- },
- arg_word
- );
-
- our_expanded_word.keys().for_each(|form| {
- let string_form: String = form.iter().collect();
- println!(" - {}", string_form);
- });
- };
-
- let should_print_old_info = already_annotated;
- let should_print_new_info =
- arg_annot.is_some() && (!already_annotated || different_annot);
-
- if should_print_old_info {
- println!("Old, from the dictionary:");
- print_word_info(
- arg_index,
- &format!("{}/{}", arg_bare_word, old_annot.unwrap()),
- );
- }
- if should_print_new_info {
- println!("New, from you:");
- print_word_info(arg_index, arg_word);
- }
- }
-
- Ok(())
- }
Args::Config => {
#[derive(Serialize)]
struct Config {
@@ -377,14 +271,3 @@ fn load_file(file: &Path, markdown_options: MarkdownOptions) -> anyhow::Result<(
Ok((Document::new_curated(&source, &parser), source))
}
-
-fn get_annotations(marked_words: &[MarkedWord], word: &str) -> Option<String> {
- for marked_word in marked_words.iter() {
- let mw = &marked_word.letters;
- let mw = mw.iter().collect::<String>();
- if mw == word {
- return Some(marked_word.attributes.iter().collect::<String>());
- }
- }
- None
-}
diff --git a/harper-comments/tests/language_support.rs b/harper-comments/tests/language_support.rs
index fc2d48b88..c0f0c7a40 100644
--- a/harper-comments/tests/language_support.rs
+++ b/harper-comments/tests/language_support.rs
@@ -47,7 +47,7 @@ create_test!(jsdoc.ts, 4);
create_test!(issue_96.lua, 0);
create_test!(merged_lines.ts, 1);
create_test!(javadoc_clean_simple.java, 0);
-create_test!(javadoc_complex.java, 4);
+create_test!(javadoc_complex.java, 5);
create_test!(issue_132.rs, 1);
create_test!(laravel_app.php, 2);
diff --git a/harper-comments/tests/language_support_sources/javadoc_clean_simple.java b/harper-comments/tests/language_support_sources/javadoc_clean_simple.java
index 54278f565..74125e621 100644
--- a/harper-comments/tests/language_support_sources/javadoc_clean_simple.java
+++ b/harper-comments/tests/language_support_sources/javadoc_clean_simple.java
@@ -1,7 +1,7 @@
class TestClass {
/**
- * This is a JavaDoc without any of the fancy frills that come with it.
+ * This is a Javadoc without any of the fancy frills that come with it.
*/
public static void main(String[] args) {
System.out.println("Hello world.");
diff --git a/harper-core/affixes.json b/harper-core/affixes.json
index 37f567396..dbd6b0d62 100644
--- a/harper-core/affixes.json
+++ b/harper-core/affixes.json
@@ -672,7 +672,9 @@
"replacements": [],
"adds_metadata": {},
"gifts_metadata": {
- "pronoun": {}
+ "pronoun": {
+ "is_plural": false
+ }
}
},
"n": {
diff --git a/harper-core/dictionary.dict b/harper-core/dictionary.dict
index 7419cb811..41dabcd27 100644
--- a/harper-core/dictionary.dict
+++ b/harper-core/dictionary.dict
@@ -4644,7 +4644,7 @@ Hymen/2M
Hyperion/2M
Hyundai/1M
Hz/M
-I/8~sf
+I/8~f
I'd/~
I'll/~
I'm/~
@@ -7388,7 +7388,6 @@ OHSA/M
OJ/15
OK/1452SMDG
OMB/2M
-ON/2
OPEC/2M
OR/712
OS/251M
@@ -26832,7 +26831,7 @@ heppest/5
heptagon/1MS
heptagonal/5
heptathlon/1SM
-her/~-8ot
+her/~-*ot
herald/~14SMDG
heralded/~4U
heraldic/~5
@@ -26929,7 +26928,7 @@ hf/~
hgt/1
hgwy/1
hi/~15D
-his/~8-
+his/~*-
hiatus/~1MS
hibachi/1MS
hibernate/4GNDS
@@ -34403,7 +34402,6 @@ nympho/1S
nymphomania/1M
nymphomaniac/15SM
nymphs/~14
-o/~15+
o'clock/
oaf/1SM
oafish/5PY
@@ -34720,8 +34718,9 @@ omniscience/1M
omniscient/51
omnivore/1MS
omnivorous/~5PY
-omnivorousness/1M
-on/~5+41Y
+omnivorousness/5
+on/~5+4
+only/~
onboard/~54
once/~71M
oncogene/1SM
@@ -48943,7 +48942,7 @@ wayside/15SM
wayward/~5PY
waywardness/1M
wazoo/1S
-we/~sf:
+we/~:
we'd/
we'll/
we're/~
@@ -50464,4 +50463,4 @@ Metasploit/21MS
cryptographic/5Q
easy peasy/5
un-American/5
-BRICS/
\ No newline at end of file
+BRICS/
diff --git a/harper-core/src/char_string.rs b/harper-core/src/char_string.rs
index ec65ac7fc..74d44efb3 100644
--- a/harper-core/src/char_string.rs
+++ b/harper-core/src/char_string.rs
@@ -9,6 +9,7 @@ pub type CharString = SmallVec<[char; 16]>;
/// Extensions to character sequences that make them easier to wrangle.
pub trait CharStringExt {
fn to_lower(&self) -> Cow<[char]>;
+ fn normalized(&self) -> Cow<[char]>;
fn to_string(&self) -> String;
}
@@ -28,6 +29,31 @@ impl CharStringExt for [char] {
fn to_string(&self) -> String {
self.iter().collect()
}
+
+ /// Convert a given character sequence to the standard character set
+ /// the dictionary is in.
+ fn normalized(&self) -> Cow<[char]> {
+ if self.as_ref().iter().any(|c| char_to_normalized(*c) != *c) {
+ Cow::Owned(
+ self.as_ref()
+ .iter()
+ .copied()
+ .map(char_to_normalized)
+ .collect(),
+ )
+ } else {
+ Cow::Borrowed(self)
+ }
+ }
+}
+
+fn char_to_normalized(c: char) -> char {
+ match c {
+ '’' => '\'',
+ '‘' => '\'',
+ ''' => '\'',
+ _ => c,
+ }
}
macro_rules! char_string {
diff --git a/harper-core/src/document.rs b/harper-core/src/document.rs
index 358d3dc37..24db1d0c7 100644
--- a/harper-core/src/document.rs
+++ b/harper-core/src/document.rs
@@ -152,7 +152,7 @@ impl Document {
/// When a word that is either an adjective or a noun is sandwiched between an article and a noun,
/// it definitely is not a noun.
fn articles_imply_nouns(&mut self) {
- let pattern = Self::ELLIPSIS_PATTERN.with(|v| v.clone());
+ let pattern = Self::ARTICLE_PATTERN.with(|v| v.clone());
for m in pattern.find_all_matches_in_doc(self) {
if let TokenKind::Word(Some(metadata)) = &mut self.tokens[m.start + 2].kind {
diff --git a/harper-core/src/lib.rs b/harper-core/src/lib.rs
index fd39b8b73..11461dc51 100644
--- a/harper-core/src/lib.rs
+++ b/harper-core/src/lib.rs
@@ -38,7 +38,7 @@ pub use mask::{Mask, Masker};
pub use number::{Number, NumberSuffix};
pub use punctuation::{Punctuation, Quote};
pub use span::Span;
-pub use spell::{Dictionary, FstDictionary, MergedDictionary, MutableDictionary};
+pub use spell::{Dictionary, FstDictionary, MergedDictionary, MutableDictionary, WordId};
pub use sync::{LSend, Lrc};
pub use title_case::{make_title_case, make_title_case_str};
pub use token::Token;
diff --git a/harper-core/src/linting/hop_hope/to_hope.rs b/harper-core/src/linting/hop_hope/to_hope.rs
index 6fd1aa5b6..e8353b80f 100644
--- a/harper-core/src/linting/hop_hope/to_hope.rs
+++ b/harper-core/src/linting/hop_hope/to_hope.rs
@@ -10,11 +10,11 @@ pub struct ToHope {
impl Default for ToHope {
fn default() -> Self {
let pattern = SequencePattern::default()
- .then_singular_subject()
+ .then_not_plural_nominal()
.then_whitespace()
.then(WordSet::new(&["hop", "hopped"]))
.then_whitespace()
- .then_singular_subject();
+ .then_nominal();
Self {
pattern: Box::new(pattern),
diff --git a/harper-core/src/linting/pique_interest.rs b/harper-core/src/linting/pique_interest.rs
index 8ab62767e..e2c128121 100644
--- a/harper-core/src/linting/pique_interest.rs
+++ b/harper-core/src/linting/pique_interest.rs
@@ -17,7 +17,7 @@ impl Default for PiqueInterest {
"peak", "peaked", "peek", "peeked", "peeking", "peaking",
]))
.then_whitespace()
- .then_singular_subject()
+ .then_not_plural_nominal()
.then_whitespace()
.t_aco("interest");
diff --git a/harper-core/src/patterns/mod.rs b/harper-core/src/patterns/mod.rs
index db5c3f028..6c60b4f97 100644
--- a/harper-core/src/patterns/mod.rs
+++ b/harper-core/src/patterns/mod.rs
@@ -25,7 +25,6 @@ mod pattern_map;
mod repeating_pattern;
mod sequence_pattern;
mod similar_to_phrase;
-mod singular_subject;
mod split_compound_word;
mod token_kind_pattern_group;
mod whitespace_pattern;
@@ -50,7 +49,6 @@ pub use pattern_map::PatternMap;
pub use repeating_pattern::RepeatingPattern;
pub use sequence_pattern::SequencePattern;
pub use similar_to_phrase::SimilarToPhrase;
-pub use singular_subject::SingularSubject;
pub use split_compound_word::SplitCompoundWord;
pub use token_kind_pattern_group::TokenKindPatternGroup;
pub use whitespace_pattern::WhitespacePattern;
diff --git a/harper-core/src/patterns/sequence_pattern.rs b/harper-core/src/patterns/sequence_pattern.rs
index 776ac7d9a..9802147b4 100644
--- a/harper-core/src/patterns/sequence_pattern.rs
+++ b/harper-core/src/patterns/sequence_pattern.rs
@@ -1,9 +1,7 @@
use paste::paste;
use super::whitespace_pattern::WhitespacePattern;
-use super::{
- AnyCapitalization, AnyPattern, IndefiniteArticle, Pattern, RepeatingPattern, SingularSubject,
-};
+use super::{AnyCapitalization, AnyPattern, IndefiniteArticle, Pattern, RepeatingPattern};
use crate::{Token, TokenKind};
/// A pattern that checks that a sequence of other patterns match.
@@ -86,6 +84,7 @@ impl SequencePattern {
gen_then_from_is!(determiner);
gen_then_from_is!(proper_noun);
gen_then_from_is!(preposition);
+ gen_then_from_is!(not_plural_nominal);
pub fn then_indefinite_article(self) -> Self {
self.then(IndefiniteArticle::default())
@@ -114,10 +113,6 @@ impl SequencePattern {
self
}
- pub fn then_singular_subject(self) -> Self {
- self.then(SingularSubject::default())
- }
-
/// Shorthand for [`Self::any_capitalization_of`].
pub fn aco(word: &'static str) -> Self {
Self::any_capitalization_of(word)
diff --git a/harper-core/src/patterns/singular_subject.rs b/harper-core/src/patterns/singular_subject.rs
deleted file mode 100644
index 4adfe1faa..000000000
--- a/harper-core/src/patterns/singular_subject.rs
+++ /dev/null
@@ -1,23 +0,0 @@
-use crate::Token;
-
-use super::{EitherPattern, OwnedPatternExt, Pattern, SequencePattern};
-
-pub struct SingularSubject {
- inner: EitherPattern,
-}
-
-impl Default for SingularSubject {
- fn default() -> Self {
- Self {
- inner: SequencePattern::default()
- .then_pronoun()
- .or(Box::new(SequencePattern::default().then_proper_noun())),
- }
- }
-}
-
-impl Pattern for SingularSubject {
- fn matches(&self, tokens: &[Token], source: &[char]) -> usize {
- self.inner.matches(tokens, source)
- }
-}
diff --git a/harper-core/src/spell/dictionary.rs b/harper-core/src/spell/dictionary.rs
index 42fb11ade..d3cde48b6 100644
--- a/harper-core/src/spell/dictionary.rs
+++ b/harper-core/src/spell/dictionary.rs
@@ -1,6 +1,7 @@
use blanket::blanket;
use super::FuzzyMatchResult;
+use super::WordId;
use crate::WordMetadata;
/// An in-memory database that contains everything necessary to parse and analyze English text.
@@ -37,12 +38,13 @@ pub trait Dictionary: Send + Sync {
/// If the word isn't in the dictionary, the resulting metadata will be
/// empty.
fn get_word_metadata_str(&self, word: &str) -> Option<&WordMetadata>;
+
/// Iterate over the words in the dictionary.
fn words_iter(&self) -> Box<dyn Iterator<Item = &'_ [char]> + Send + '_>;
/// The number of words in the dictionary.
fn word_count(&self) -> usize;
- /// Iterate over all the words in the dictionary of a given length
- fn words_with_len_iter(&self, len: usize) -> Box<dyn Iterator<Item = &'_ [char]> + Send + '_>;
+ /// Returns the correct capitalization of the word with the given ID.
+ fn get_word_from_id(&self, id: &WordId) -> Option<&[char]>;
}
diff --git a/harper-core/src/spell/fst_dictionary.rs b/harper-core/src/spell/fst_dictionary.rs
index e637eeb3c..f7ddf8013 100644
--- a/harper-core/src/spell/fst_dictionary.rs
+++ b/harper-core/src/spell/fst_dictionary.rs
@@ -1,10 +1,9 @@
use super::{
- MutableDictionary,
+ MutableDictionary, WordId,
hunspell::{parse_default_attribute_list, parse_default_word_list},
- seq_to_normalized,
+ word_map::WordMap,
};
use fst::{IntoStreamer, Map as FstMap, Streamer, map::StreamWithState};
-use hashbrown::HashMap;
use lazy_static::lazy_static;
use levenshtein_automata::{DFA, LevenshteinAutomatonBuilder};
use std::{cell::RefCell, sync::Arc};
@@ -34,10 +33,15 @@ fn uncached_inner_new() -> Arc<FstDictionary> {
let attr_list = parse_default_attribute_list();
// There will be at _least_ this number of words
- let mut word_map = HashMap::with_capacity(word_list.len());
+ let mut word_map = WordMap::with_capacity(word_list.len());
attr_list.expand_marked_words(word_list, &mut word_map);
- Arc::new(FstDictionary::new(word_map))
+ Arc::new(FstDictionary::new(
+ word_map
+ .into_iter()
+ .map(|entry| (entry.canonical_spelling, entry.metadata))
+ .collect(),
+ ))
}
const EXPECTED_DISTANCE: u8 = 3;
@@ -71,8 +75,7 @@ impl FstDictionary {
(*DICT).clone()
}
- pub fn new(new_words: HashMap<CharString, WordMetadata>) -> Self {
- let mut words: Vec<(CharString, WordMetadata)> = new_words.into_iter().collect();
+ pub fn new(mut words: Vec<(CharString, WordMetadata)>) -> Self {
words.sort_unstable_by(|(a, _), (b, _)| a.cmp(b));
words.dedup_by(|(a, _), (b, _)| a == b);
@@ -151,7 +154,7 @@ impl Dictionary for FstDictionary {
max_distance: u8,
max_results: usize,
) -> Vec<FuzzyMatchResult> {
- let misspelled_word_charslice = seq_to_normalized(word);
+ let misspelled_word_charslice = word.normalized();
let misspelled_word_string = misspelled_word_charslice.to_string();
// Actual FST search
@@ -210,10 +213,6 @@ impl Dictionary for FstDictionary {
self.full_dict.words_iter()
}
- fn words_with_len_iter(&self, len: usize) -> Box<dyn Iterator<Item = &'_ [char]> + Send + '_> {
- self.full_dict.words_with_len_iter(len)
- }
-
fn word_count(&self) -> usize {
self.full_dict.word_count()
}
@@ -229,6 +228,10 @@ impl Dictionary for FstDictionary {
fn get_correct_capitalization_of(&self, word: &[char]) -> Option<&'_ [char]> {
self.full_dict.get_correct_capitalization_of(word)
}
+
+ fn get_word_from_id(&self, id: &WordId) -> Option<&[char]> {
+ self.full_dict.get_word_from_id(id)
+ }
}
#[cfg(test)]
@@ -236,7 +239,7 @@ mod tests {
use itertools::Itertools;
use crate::CharStringExt;
- use crate::{Dictionary, spell::seq_to_normalized};
+ use crate::Dictionary;
use super::FstDictionary;
@@ -245,7 +248,7 @@ mod tests {
let dict = FstDictionary::curated();
for word in dict.words_iter() {
- let misspelled_normalized = seq_to_normalized(word);
+ let misspelled_normalized = word.normalized();
let misspelled_word = misspelled_normalized.to_string();
let misspelled_lower = misspelled_normalized.to_lower().to_string();
@@ -264,9 +267,9 @@ mod tests {
let dict = FstDictionary::curated();
let word: Vec<_> = "hello".chars().collect();
- let misspelled_normalized = seq_to_normalized(&word);
- let misspelled_word: String = misspelled_normalized.to_string();
- let misspelled_lower: String = misspelled_normalized.to_lower().to_string();
+ let misspelled_normalized = word.normalized();
+ let misspelled_word = misspelled_normalized.to_string();
+ let misspelled_lower = misspelled_normalized.to_lower().to_string();
assert!(dict.contains_word(&misspelled_normalized));
assert!(
@@ -275,6 +278,13 @@ mod tests {
);
}
+ #[test]
+ fn on_is_not_nominal() {
+ let dict = FstDictionary::curated();
+
+ assert!(!dict.get_word_metadata_str("on").unwrap().is_nominal());
+ }
+
#[test]
fn fuzzy_result_sorted_by_edit_distance() {
let dict = FstDictionary::curated();
diff --git a/harper-core/src/spell/hunspell/attribute_list.rs b/harper-core/src/spell/hunspell/attribute_list.rs
index d84f2ffc8..ef637cbda 100644
--- a/harper-core/src/spell/hunspell/attribute_list.rs
+++ b/harper-core/src/spell/hunspell/attribute_list.rs
@@ -2,6 +2,7 @@ use hashbrown::HashMap;
use serde::{Deserialize, Serialize};
use smallvec::ToSmallVec;
+use super::super::word_map::{WordMap, WordMapEntry};
use super::Error;
use super::affix_replacement::AffixReplacement;
use super::expansion::{Expansion, HumanReadableExpansion};
@@ -30,11 +31,7 @@ impl AttributeList {
/// Will append to the given `dest`;
///
/// In the future, I want to make this function cleaner and faster.
- pub fn expand_marked_word(
- &self,
- word: MarkedWord,
- dest: &mut HashMap<CharString, WordMetadata>,
- ) {
+ pub fn expand_marked_word(&self, word: MarkedWord, dest: &mut WordMap) {
dest.reserve(word.attributes.len() + 1);
let mut gifted_metadata = WordMetadata::default();
@@ -78,24 +75,33 @@ impl AttributeList {
},
dest,
);
- let t_metadata = dest.get_mut(&new_word).unwrap();
+ let t_metadata = dest.get_metadata_mut_chars(&new_word).unwrap();
t_metadata.append(&metadata);
}
} else {
for (key, value) in new_words.into_iter() {
- if let Some(val) = dest.get_mut(&key) {
+ if let Some(val) = dest.get_metadata_mut_chars(&key) {
val.append(&value);
} else {
- dest.insert(key, value);
+ dest.insert(WordMapEntry {
+ canonical_spelling: key,
+ metadata: value,
+ });
}
}
}
}
- if let Some(prev_val) = dest.get(&word.letters) {
- dest.insert(word.letters, gifted_metadata.or(prev_val));
+ if let Some(prev_val) = dest.get_with_chars(&word.letters) {
+ dest.insert(WordMapEntry {
+ metadata: gifted_metadata.or(&prev_val.metadata),
+ canonical_spelling: word.letters,
+ });
} else {
- dest.insert(word.letters, gifted_metadata);
+ dest.insert(WordMapEntry {
+ metadata: gifted_metadata,
+ canonical_spelling: word.letters,
+ });
}
}
@@ -105,7 +111,7 @@ impl AttributeList {
pub fn expand_marked_words(
&self,
words: impl IntoIterator<Item = MarkedWord>,
- dest: &mut HashMap<CharString, WordMetadata>,
+ dest: &mut WordMap,
) {
for word in words {
self.expand_marked_word(word, dest);
diff --git a/harper-core/src/spell/hunspell/mod.rs b/harper-core/src/spell/hunspell/mod.rs
index 22e038305..518ce498f 100644
--- a/harper-core/src/spell/hunspell/mod.rs
+++ b/harper-core/src/spell/hunspell/mod.rs
@@ -28,13 +28,14 @@ pub fn parse_default_attribute_list() -> AttributeList {
#[cfg(test)]
mod tests {
- use hashbrown::{HashMap, HashSet};
+ use hashbrown::HashSet;
use serde_json::json;
+ use super::super::word_map::WordMap;
use super::word_list::parse_word_list;
use super::{parse_default_attribute_list, parse_default_word_list};
+ use crate::CharStringExt;
use crate::spell::hunspell::attribute_list::HumanReadableAttributeList;
- use crate::{CharString, WordMetadata};
pub const TEST_WORD_LIST: &str = "3\nhello\ntry/B\nwork/AB";
@@ -85,12 +86,12 @@ mod tests {
.unwrap();
let attributes = attributes.into_normal().unwrap();
- let mut expanded = HashMap::new();
+ let mut expanded = WordMap::default();
attributes.expand_marked_words(words, &mut expanded);
let expanded: HashSet<String> = expanded
.into_iter()
- .map(|v| v.0.into_iter().collect())
+ .map(|v| v.canonical_spelling.to_string())
.collect();
assert_eq!(
@@ -157,22 +158,22 @@ mod tests {
.unwrap();
let attributes = attributes.into_normal().unwrap();
- let mut expanded: HashMap<CharString, WordMetadata> = HashMap::new();
+ let mut expanded = WordMap::default();
attributes.expand_marked_words(words, &mut expanded);
- let giant_data = expanded.get(&split("giant")).unwrap();
- assert!(giant_data.is_noun());
+ let giant_data = expanded.get_with_str("giant").unwrap();
+ assert!(giant_data.metadata.is_noun());
- let giants_data = expanded.get(&split("giants")).unwrap();
- assert!(giants_data.is_plural_noun());
+ let giants_data = expanded.get_with_str("giants").unwrap();
+ assert!(giants_data.metadata.is_plural_noun());
}
- fn build_expanded() -> HashMap<CharString, WordMetadata> {
+ fn build_expanded() -> WordMap {
let words = parse_default_word_list().unwrap();
let attributes = parse_default_attribute_list();
- let mut expanded = HashMap::new();
+ let mut expanded = WordMap::default();
attributes.expand_marked_words(words, &mut expanded);
@@ -186,29 +187,30 @@ mod tests {
#[test]
fn expanded_contains_giants() {
- assert!(build_expanded().contains_key(&split("giants")));
+ assert!(build_expanded().contains_str("giants"));
}
#[test]
fn expanded_contains_deallocate() {
- assert!(build_expanded().contains_key(&split("deallocate")));
+ assert!(build_expanded().contains_str("deallocate"));
}
#[test]
fn expanded_contains_repo() {
let expanded = build_expanded();
- assert!(expanded.contains_key(&split("repo")));
- assert!(expanded.contains_key(&split("repos")));
- assert!(expanded.contains_key(&split("repo's")));
+ assert!(expanded.contains_str("repo"));
+ assert!(expanded.contains_str("repos"));
+ assert!(expanded.contains_str("repo's"));
}
#[test]
fn expanded_contains_possessive_abandonment() {
assert!(
build_expanded()
- .get(&split("abandonment's"))
+ .get_with_str("abandonment's")
.unwrap()
+ .metadata
.is_possessive_noun()
)
}
@@ -217,21 +219,21 @@ mod tests {
fn has_is_not_a_nominal() {
let expanded = build_expanded();
- let has = expanded.get(&split("has"));
+ let has = expanded.get_with_str("has");
assert!(has.is_some());
- assert!(!has.unwrap().is_nominal(),)
+ assert!(!has.unwrap().metadata.is_nominal())
}
#[test]
fn is_is_linking_verb() {
let expanded = build_expanded();
- let is = expanded.get(&split("is"));
+ let is = expanded.get_with_str("is");
dbg!(&is);
assert!(is.is_some());
- assert!(is.unwrap().is_linking_verb());
+ assert!(is.unwrap().metadata.is_linking_verb());
}
#[test]
@@ -242,19 +244,19 @@ mod tests {
let merged_attrs = parse_default_attribute_list();
let spread_attrs = parse_default_attribute_list();
- let mut expanded1 = HashMap::new();
- let mut expanded2 = HashMap::new();
+ let mut expanded1 = WordMap::default();
+ let mut expanded2 = WordMap::default();
merged_attrs.expand_marked_words(merged_word, &mut expanded1);
let expanded_merged: HashSet<String> = expanded1
.into_iter()
- .map(|v| v.0.into_iter().collect())
+ .map(|v| v.canonical_spelling.into_iter().collect())
.collect();
spread_attrs.expand_marked_words(spread_word, &mut expanded2);
let expanded_spread: HashSet<String> = expanded2
.into_iter()
- .map(|v| v.0.into_iter().collect())
+ .map(|v| v.canonical_spelling.into_iter().collect())
.collect();
assert_eq!(
@@ -262,8 +264,4 @@ mod tests {
expanded_spread.into_iter().collect::<HashSet<_>>()
);
}
-
- fn split(text: &str) -> CharString {
- text.chars().collect()
- }
}
diff --git a/harper-core/src/spell/merged_dictionary.rs b/harper-core/src/spell/merged_dictionary.rs
index ce04ac54c..5792db00a 100644
--- a/harper-core/src/spell/merged_dictionary.rs
+++ b/harper-core/src/spell/merged_dictionary.rs
@@ -4,7 +4,7 @@ use std::sync::Arc;
use foldhash::quality::FixedState;
use itertools::Itertools;
-use super::FstDictionary;
+use super::{FstDictionary, WordId};
use super::{FuzzyMatchResult, dictionary::Dictionary};
use crate::{CharString, WordMetadata};
@@ -107,14 +107,6 @@ impl Dictionary for MergedDictionary {
Box::new(self.children.iter().flat_map(|c| c.words_iter()))
}
- fn words_with_len_iter(&self, len: usize) -> Box<dyn Iterator<Item = &'_ [char]> + Send + '_> {
- Box::new(
- self.children
- .iter()
- .flat_map(move |c| c.words_with_len_iter(len)),
- )
- }
-
fn contains_word_str(&self, word: &str) -> bool {
let chars: CharString = word.chars().collect();
self.contains_word(&chars)
@@ -161,4 +153,10 @@ impl Dictionary for MergedDictionary {
fn word_count(&self) -> usize {
self.children.iter().map(|d| d.word_count()).sum()
}
+
+ fn get_word_from_id(&self, id: &WordId) -> Option<&[char]> {
+ self.children
+ .iter()
+ .find_map(|dict| dict.get_word_from_id(id))
+ }
}
diff --git a/harper-core/src/spell/mod.rs b/harper-core/src/spell/mod.rs
index c53364c22..1db80ef74 100644
--- a/harper-core/src/spell/mod.rs
+++ b/harper-core/src/spell/mod.rs
@@ -1,17 +1,18 @@
-use std::borrow::Cow;
-
use crate::{CharString, CharStringExt, WordMetadata};
pub use self::dictionary::Dictionary;
pub use self::fst_dictionary::FstDictionary;
pub use self::merged_dictionary::MergedDictionary;
pub use self::mutable_dictionary::MutableDictionary;
+pub use self::word_id::WordId;
mod dictionary;
mod fst_dictionary;
pub mod hunspell;
mod merged_dictionary;
mod mutable_dictionary;
+mod word_id;
+mod word_map;
#[derive(PartialEq, Debug, Hash, Eq)]
pub struct FuzzyMatchResult<'a> {
@@ -95,30 +96,13 @@ pub fn suggest_correct_spelling_str(
.collect()
}
-/// Convert a given character sequence to the standard character set
-/// the dictionary is in.
-fn seq_to_normalized(seq: &[char]) -> Cow<'_, [char]> {
- if seq.iter().any(|c| char_to_normalized(*c) != *c) {
- Cow::Owned(seq.iter().copied().map(char_to_normalized).collect())
- } else {
- Cow::Borrowed(seq)
- }
-}
-
-fn char_to_normalized(c: char) -> char {
- match c {
- '’' => '\'',
- '‘' => '\'',
- ''' => '\'',
- _ => c,
- }
-}
-
#[cfg(test)]
mod tests {
use itertools::Itertools;
- use super::{FstDictionary, seq_to_normalized, suggest_correct_spelling_str};
+ use crate::CharStringExt;
+
+ use super::{FstDictionary, suggest_correct_spelling_str};
const RESULT_LIMIT: usize = 100;
const MAX_EDIT_DIST: u8 = 2;
@@ -126,7 +110,7 @@ mod tests {
#[test]
fn normalizes_weve() {
let word = vec!['w', 'e', '’', 'v', 'e'];
- let norm = seq_to_normalized(&word);
+ let norm = word.normalized();
assert_eq!(norm.clone(), vec!['w', 'e', '\'', 'v', 'e'])
}
diff --git a/harper-core/src/spell/mutable_dictionary.rs b/harper-core/src/spell/mutable_dictionary.rs
index 5a35f1e09..15f46dd7b 100644
--- a/harper-core/src/spell/mutable_dictionary.rs
+++ b/harper-core/src/spell/mutable_dictionary.rs
@@ -1,12 +1,11 @@
use super::{
+ WordId,
hunspell::{parse_default_attribute_list, parse_default_word_list},
- seq_to_normalized,
+ word_map::{WordMap, WordMapEntry},
};
use crate::edit_distance::edit_distance_min_alloc;
-use hashbrown::HashMap;
use itertools::Itertools;
use lazy_static::lazy_static;
-use smallvec::{SmallVec, ToSmallVec};
use std::sync::Arc;
use crate::{CharString, CharStringExt, WordMetadata};
@@ -24,25 +23,8 @@ use super::dictionary::Dictionary;
/// [`super::MergedDictionary`].
#[derive(Debug, Clone, Eq, PartialEq)]
pub struct MutableDictionary {
- /// Storing a separate [`Vec`] for iterations speeds up spellchecking by
- /// ~16% at the cost of additional memory.
- ///
- /// This is likely due to increased locality 🤷.
- ///
- /// This list is sorted by word length (i.e. the shortest words are first).
- words: Vec<CharString>,
- /// A lookup list for each word length.
- /// Each index of this list will return the first index of [`Self::words`]
- /// that has a word whose index is that length.
- word_len_starts: Vec<usize>,
/// All English words
- word_map: HashMap<CharString, WordMetadata>,
- /// A map from the lowercase versions of a word to the correct capitalization
- /// of that same word.
- ///
- /// It can be used to check if a word is correctly capitalized, or if it is valid, regardless of
- /// capitalization.
- word_map_lowercase: HashMap<CharString, CharString>,
+ word_map: WordMap,
}
/// The uncached function that is used to produce the original copy of the
@@ -52,27 +34,11 @@ fn uncached_inner_new() -> Arc<MutableDictionary> {
let attr_list = parse_default_attribute_list();
// There will be at _least_ this number of words
- let mut word_map = HashMap::with_capacity(word_list.len());
+ let mut word_map = WordMap::default();
attr_list.expand_marked_words(word_list, &mut word_map);
- let mut words: Vec<CharString> = word_map.iter().map(|(v, _)| v.clone()).collect();
-
- words.sort_unstable();
- words.dedup();
- words.sort_unstable_by_key(|w| w.len());
-
- let mut word_map_lowercase = HashMap::with_capacity(word_map.len());
- for key in word_map.keys() {
- word_map_lowercase.insert(key.to_lower().to_smallvec(), key.clone());
- }
-
- Arc::new(MutableDictionary {
- word_map,
- word_map_lowercase,
- word_len_starts: MutableDictionary::create_len_starts(&words),
- words,
- })
+ Arc::new(MutableDictionary { word_map })
}
lazy_static! {
@@ -82,10 +48,7 @@ lazy_static! {
impl MutableDictionary {
pub fn new() -> Self {
Self {
- words: Vec::new(),
- word_len_starts: Vec::new(),
- word_map: HashMap::new(),
- word_map_lowercase: HashMap::new(),
+ word_map: WordMap::default(),
}
}
@@ -103,22 +66,12 @@ impl MutableDictionary {
&mut self,
words: impl IntoIterator<Item = (impl AsRef<[char]>, WordMetadata)>,
) {
- let pairs: Vec<_> = words
- .into_iter()
- .filter_map(|(v, m)| {
- (!self.contains_word(v.as_ref())).then(|| (v.as_ref().to_smallvec(), m))
+ for (chars, metadata) in words.into_iter() {
+ self.word_map.insert(WordMapEntry {
+ metadata,
+ canonical_spelling: chars.as_ref().into(),
})
- .collect();
-
- self.words.extend(pairs.iter().map(|(v, _)| v.clone()));
- self.words.sort_by_key(|w| w.len());
- self.word_len_starts = Self::create_len_starts(&self.words);
- self.word_map_lowercase.extend(
- pairs
- .iter()
- .map(|(key, _)| (key.to_lower().to_smallvec(), key.clone())),
- );
- self.word_map.extend(pairs);
+ }
}
/// Append a single word to the dictionary.
@@ -136,24 +89,6 @@ impl MutableDictionary {
pub fn append_word_str(&mut self, word: &str, metadata: WordMetadata) {
self.append_word(word.chars().collect::<Vec<_>>(), metadata)
}
-
- /// Create a lookup table for finding words of a specific length in a word
- /// list.
- fn create_len_starts(words: &[CharString]) -> Vec<usize> {
- let mut len_words: Vec<_> = words.to_vec();
- len_words.sort_by_key(|a| a.len());
-
- let mut word_len_starts = vec![0, 0];
-
- for (index, len) in len_words.iter().map(SmallVec::len).enumerate() {
- if word_len_starts.len() <= len {
- word_len_starts.resize(len, index);
- word_len_starts.push(index);
- }
- }
-
- word_len_starts
- }
}
impl Default for MutableDictionary {
@@ -164,17 +99,11 @@ impl Default for MutableDictionary {
impl Dictionary for MutableDictionary {
fn get_word_metadata(&self, word: &[char]) -> Option<&WordMetadata> {
- let normalized = seq_to_normalized(word);
- let correct_caps = self.get_correct_capitalization_of(&normalized)?;
-
- self.word_map.get(correct_caps)
+ self.word_map.get_with_chars(word).map(|v| &v.metadata)
}
fn contains_word(&self, word: &[char]) -> bool {
- let normalized = seq_to_normalized(word);
- let lowercase = normalized.to_lower();
-
- self.word_map_lowercase.contains_key(lowercase.as_ref())
+ self.word_map.contains_chars(word)
}
fn contains_word_str(&self, word: &str) -> bool {
@@ -188,12 +117,9 @@ impl Dictionary for MutableDictionary {
}
fn get_correct_capitalization_of(&self, word: &[char]) -> Option<&'_ [char]> {
- let normalized = seq_to_normalized(word);
- let lowercase = normalized.to_lower();
-
- self.word_map_lowercase
- .get(lowercase.as_ref())
- .map(|v| v.as_slice())
+ self.word_map
+ .get_with_chars(word)
+ .map(|v| v.canonical_spelling.as_slice())
}
/// Suggest a correct spelling for a given misspelled word.
@@ -206,7 +132,7 @@ impl Dictionary for MutableDictionary {
max_distance: u8,
max_results: usize,
) -> Vec<FuzzyMatchResult> {
- let misspelled_charslice = seq_to_normalized(word);
+ let misspelled_charslice = word.normalized();
let misspelled_charslice_lower = misspelled_charslice.to_lower();
let shortest_word_len = if misspelled_charslice.len() <= max_distance as usize {
@@ -217,9 +143,9 @@ impl Dictionary for MutableDictionary {
let longest_word_len = misspelled_charslice.len() + max_distance as usize;
// Get candidate words
- let words_to_search = (shortest_word_len..=longest_word_len)
- .rev()
- .flat_map(|len| self.words_with_len_iter(len));
+ let words_to_search = self
+ .words_iter()
+ .filter(|word| (shortest_word_len..=longest_word_len).contains(&word.len()));
// Pre-allocated vectors for the edit-distance calculation
// 53 is the length of the longest word.
@@ -266,36 +192,37 @@ impl Dictionary for MutableDictionary {
}
fn words_iter(&self) -> Box<dyn Iterator<Item = &'_ [char]> + Send + '_> {
- Box::new(self.words.iter().map(|v| v.as_slice()))
- }
-
- fn words_with_len_iter(&self, len: usize) -> Box<dyn Iterator<Item = &'_ [char]> + Send + '_> {
- if len == 0 || len >= self.word_len_starts.len() {
- return Box::new(std::iter::empty());
- }
-
- let start = self.word_len_starts[len];
- let end = if len + 1 == self.word_len_starts.len() {
- self.words.len()
- } else {
- self.word_len_starts[len + 1]
- };
-
- Box::new(self.words[start..end].iter().map(|v| v.as_slice()))
+ Box::new(
+ self.word_map
+ .iter()
+ .map(|v| v.canonical_spelling.as_slice()),
+ )
}
fn word_count(&self) -> usize {
- self.words.len()
+ self.word_map.len()
}
fn contains_exact_word(&self, word: &[char]) -> bool {
- self.word_map.contains_key(seq_to_normalized(word).as_ref())
+ let normalized = word.normalized();
+
+ if let Some(found) = self.word_map.get_with_chars(normalized.as_ref()) {
+ if found.canonical_spelling.as_ref() == normalized.as_ref() {
+ return true;
+ }
+ }
+
+ false
}
fn contains_exact_word_str(&self, word: &str) -> bool {
let word: CharString = word.chars().collect();
self.contains_exact_word(word.as_ref())
}
+
+ fn get_word_from_id(&self, id: &WordId) -> Option<&[char]> {
+ self.word_map.get(id).map(|w| w.canonical_spelling.as_ref())
+ }
}
#[cfg(test)]
@@ -305,19 +232,10 @@ mod tests {
use crate::{Dictionary, MutableDictionary};
- #[test]
- fn words_with_len_contains_self() {
- let dict = MutableDictionary::curated();
-
- let word: CharString = "hello".chars().collect();
- let words_with_same_len = dict.words_with_len_iter(word.len()).collect_vec();
- assert!(words_with_same_len.contains(&&word[..]));
- }
-
#[test]
fn curated_contains_no_duplicates() {
let dict = MutableDictionary::curated();
- assert!(dict.words.iter().all_unique());
+ assert!(dict.words_iter().all_unique());
}
#[test]
diff --git a/harper-core/src/spell/word_id.rs b/harper-core/src/spell/word_id.rs
new file mode 100644
index 000000000..d889f9c3e
--- /dev/null
+++ b/harper-core/src/spell/word_id.rs
@@ -0,0 +1,26 @@
+use std::hash::BuildHasher;
+
+use foldhash::fast::FixedState;
+
+use crate::CharStringExt;
+
+/// An identifier for a particular word.
+///
+/// It works by hashing the word it represents, normalized to lowercase.
+/// It is meant for situations where you need to refer to a word (or a collection of words),
+/// without storing all of accompanying data (like spelling or metadata).
+#[derive(Hash, Copy, Clone, PartialEq, Eq, PartialOrd, Debug)]
+pub struct WordId {
+ hash: u64,
+}
+
+impl WordId {
+ /// Create a Word ID from
+ pub fn from_word_chars(chars: impl AsRef<[char]>) -> Self {
+ let normalized = chars.as_ref().normalized();
+ let lower = normalized.to_lower();
+ let hash = FixedState::default().hash_one(lower);
+
+ Self { hash }
+ }
+}
diff --git a/harper-core/src/spell/word_map.rs b/harper-core/src/spell/word_map.rs
new file mode 100644
index 000000000..66a40f2f3
--- /dev/null
+++ b/harper-core/src/spell/word_map.rs
@@ -0,0 +1,104 @@
+use hashbrown::{HashMap, hash_map::IntoValues};
+
+use crate::{CharString, WordMetadata};
+
+use super::WordId;
+
+/// The underlying data structure for the `MutableDictionary`.
+#[derive(Debug, Clone, Eq, PartialEq, Default)]
+pub struct WordMap {
+ inner: HashMap<WordId, WordMapEntry>,
+}
+
+#[derive(Debug, Clone, Eq, PartialEq)]
+pub struct WordMapEntry {
+ pub metadata: WordMetadata,
+ pub canonical_spelling: CharString,
+}
+
+impl WordMap {
+ /// Get an entry from the word map using raw chars.
+ pub fn get_with_str(&self, string: &str) -> Option<&WordMapEntry> {
+ let chars: CharString = string.chars().collect();
+ let id = WordId::from_word_chars(chars);
+
+ self.get(&id)
+ }
+
+ pub fn contains_str(&self, string: &str) -> bool {
+ self.get_with_str(string).is_some()
+ }
+
+ pub fn contains_chars(&self, chars: impl AsRef<[char]>) -> bool {
+ self.get_with_chars(chars).is_some()
+ }
+
+ pub fn contains(&self, id: &WordId) -> bool {
+ self.get(id).is_some()
+ }
+
+ /// Get an entry from the word map using raw chars.
+ pub fn get_with_chars(&self, chars: impl AsRef<[char]>) -> Option<&WordMapEntry> {
+ let id = WordId::from_word_chars(chars);
+
+ self.get(&id)
+ }
+
+ /// Get an entry from the word map using a word identifier.
+ pub fn get(&self, id: &WordId) -> Option<&WordMapEntry> {
+ self.inner.get(id)
+ }
+
+ /// Borrow a word's metadata mutably
+ pub fn get_metadata_mut_chars(
+ &mut self,
+ chars: impl AsRef<[char]>,
+ ) -> Option<&mut WordMetadata> {
+ let id = WordId::from_word_chars(chars);
+
+ self.get_metadata_mut(&id)
+ }
+
+ /// Borrow a word's metadata mutably
+ pub fn get_metadata_mut(&mut self, id: &WordId) -> Option<&mut WordMetadata> {
+ self.inner.get_mut(id).map(|v| &mut v.metadata)
+ }
+
+ pub fn insert(&mut self, entry: WordMapEntry) {
+ let id = WordId::from_word_chars(&entry.canonical_spelling);
+
+ self.inner.insert(id, entry);
+ }
+
+ /// Reserves capacity for at least `additional` more elements to be inserted
+ /// in the `WordMap`. The collection may reserve more space to avoid
+ /// frequent reallocations.
+ pub fn reserve(&mut self, additional: usize) {
+ self.inner.reserve(additional);
+ }
+
+ /// Iterate through the canonical spellings of the words in the map.
+ pub fn iter(&self) -> impl Iterator<Item = &WordMapEntry> {
+ self.inner.values()
+ }
+
+ pub fn len(&self) -> usize {
+ self.inner.len()
+ }
+
+ pub fn with_capacity(capacity: usize) -> Self {
+ Self {
+ inner: HashMap::with_capacity(capacity),
+ }
+ }
+}
+
+impl IntoIterator for WordMap {
+ type Item = WordMapEntry;
+
+ fn into_iter(self) -> Self::IntoIter {
+ self.inner.into_values()
+ }
+
+ type IntoIter = IntoValues<WordId, WordMapEntry>;
+}
diff --git a/harper-core/src/word_metadata.rs b/harper-core/src/word_metadata.rs
index 3bc1f9862..74bfc06dc 100644
--- a/harper-core/src/word_metadata.rs
+++ b/harper-core/src/word_metadata.rs
@@ -2,6 +2,8 @@ use is_macro::Is;
use paste::paste;
use serde::{Deserialize, Serialize};
+use crate::WordId;
+
#[derive(Debug, Default, Clone, PartialEq, Eq, Serialize, Deserialize, PartialOrd, Hash)]
pub struct WordMetadata {
pub noun: Option<NounData>,
@@ -24,6 +26,8 @@ pub struct WordMetadata {
/// Whether the word is considered especially common.
#[serde(default = "default_false")]
pub common: bool,
+ #[serde(skip_serializing, skip_deserializing)]
+ pub derived_from: Option<WordId>,
}
/// Needed for `serde`
@@ -105,6 +109,7 @@ impl WordMetadata {
determiner: self.determiner || other.determiner,
preposition: self.preposition || other.preposition,
common: self.common || other.common,
+ derived_from: self.derived_from.or(other.derived_from),
}
}
@@ -164,7 +169,7 @@ impl WordMetadata {
is_plural: Some(false),
..
})
- ) && matches!(
+ ) || matches!(
self.pronoun,
Some(PronounData {
is_plural: Some(false),
@@ -195,7 +200,7 @@ impl WordMetadata {
matches!(self.swear, Some(true))
}
- /// Same thing as [`Self::or`], except in-place rather than a copy.
+ /// Same thing as [`Self::or`], except in-place rather than a clone.
pub fn append(&mut self, other: &Self) -> &mut Self {
*self = self.or(other);
self