Offline, privacy-first grammar checker. Fast, open-source, Rust-powered
feat(core)!: more explicit handling of case-sensitivity in dictionaries (#2630)
Steve Kurch760378b7ab40
diff --git a/Cargo.lock b/Cargo.lockindex c2ac994e2..e06d66827 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -2724,6 +2724,7 @@ dependencies = ["harper-brill","harper-thesaurus","hashbrown 0.16.1",+ "indexmap","is-macro","itertools 0.15.0","levenshtein_automata",diff --git a/harper-cli/src/main.rs b/harper-cli/src/main.rsindex 519b1f09d..2d314b5cc 100644--- a/harper-cli/src/main.rs+++ b/harper-cli/src/main.rs@@ -21,7 +21,7 @@ use harper_core::{CharStringExt, Dialect, DictWordMetadata, OrthFlags, Span, TokenKind, TokenStringExt,linting::LintGroup,parsers::{IsolateEnglish, MarkdownOptions},- spell::{Dictionary, FstDictionary, MutableDictionary, WordId},+ spell::{CanonicalWordId, Dictionary, FstDictionary, MutableDictionary},weir::WeirLinter,};#[cfg(feature = "training")]@@ -1065,11 +1065,15 @@ fn line_to_parts(line: &str) -> (String, String) {fn print_word_derivations(word: &str, annot: &str, dictionary: &impl Dictionary) {println!("{word}/{annot}");- let id = WordId::from_word_str(word);+ let id = CanonicalWordId::from_word_str(word);- let children = dictionary- .words_iter()- .filter(|e| dictionary.get_word_metadata(e).unwrap().derived_from == Some(id));+ let children = dictionary.words_iter().filter(|word| {+ dictionary+ .get_word_metadata_exact(word)+ .unwrap()+ .derived_from+ .contains(id)+ });println!(" - {word}");diff --git a/harper-core/Cargo.toml b/harper-core/Cargo.tomlindex 33fff3434..bd3b0d8e0 100644--- a/harper-core/Cargo.toml+++ b/harper-core/Cargo.toml@@ -34,6 +34,7 @@ harper-brill = { path = "../harper-brill", version = "2.0.0" }harper-thesaurus = { path = "../harper-thesaurus", version = "2.0.0", optional = true }bitflags = { version = "2.11.0", features = ["serde"] }trie-rs = "0.4.2"+indexmap = "2.13.1"zip = { version = "8.6.0", default-features = false, features = ["deflate"] }regex = "1.12.3"boxcar = { version = "0.2.14", optional = true }diff --git a/harper-core/dictionary.dict b/harper-core/dictionary.dictindex 94247fcb6..53f4670b1 100644--- a/harper-core/dictionary.dict+++ b/harper-core/dictionary.dict@@ -8311,6 +8311,7 @@ PowerPoint/ONgVPowers/NOgPowhatan/NOgPoznan/Og+Pr/ # PraseodymiumPrada/gPrado/OgPraetorian/Ngdiff --git a/harper-core/src/dict_word_metadata.rs b/harper-core/src/dict_word_metadata.rsindex acbebdf80..98505caf6 100644--- a/harper-core/src/dict_word_metadata.rs+++ b/harper-core/src/dict_word_metadata.rs@@ -10,9 +10,11 @@ use strum_macros::{Display, EnumCount, EnumIter, EnumString, VariantArray};use std::convert::TryFrom;use crate::dict_word_metadata_orthography::OrthFlags;-use crate::spell::WordId;use crate::{Document, TokenKind, TokenStringExt};+pub mod derived_from;+use derived_from::DerivedFrom;+/// This represents a "lexeme" or "headword" which is case-folded but affix-expanded./// So not only lemmata but also inflected forms are stored here, with "horn" and "horns" each/// having their own lexeme, but "Ivy" and "ivy" sharing the same lexeme.@@ -46,8 +48,8 @@ pub struct DictWordMetadata {/// Whether the word is considered especially common.#[serde(default = "default_false")]pub common: bool,- #[serde(default = "default_none")]- pub derived_from: Option<WordId>,+ #[serde(default = "DerivedFrom::default")]+ pub derived_from: DerivedFrom,/// Generated by a chunker. Declares whether the word is a member of a nominal phrase. Using/// this should be preferred over the similarly named `Pattern`.///@@ -62,11 +64,6 @@ fn default_false() -> bool {false}-/// Needed for `serde`-fn default_none<T>() -> Option<T> {- None-}-/// Needed for `serde`fn default_default<T: Default>() -> T {T::default()@@ -863,7 +860,7 @@ impl DictWordMetadata {self.swear = self.swear.or(other.swear);self.abbreviation = self.abbreviation.or(other.abbreviation);self.common |= other.common;- self.derived_from = self.derived_from.or(other.derived_from);+ self.derived_from.extend(other.derived_from.iter());self.pos_tag = self.pos_tag.or(other.pos_tag);self.np_member = self.np_member.or(other.np_member);diff --git a/harper-core/src/dict_word_metadata/derived_from.rs b/harper-core/src/dict_word_metadata/derived_from.rsnew file mode 100644index 000000000..044de1b04--- /dev/null+++ b/harper-core/src/dict_word_metadata/derived_from.rs@@ -0,0 +1,73 @@+use std::{+ collections::{BTreeSet, btree_set},+ iter::Extend,+};++use serde::{Deserialize, Serialize};++use crate::spell::CanonicalWordId;++/// A container for storing word IDs that a word is considered to be derived from.+#[derive(Debug, Default, Clone, PartialEq, Eq, Serialize, Deserialize, PartialOrd, Hash)]+pub struct DerivedFrom {+ inner: BTreeSet<CanonicalWordId>,+}++impl DerivedFrom {+ /// Insert another word ID, if it's not already contained.+ ///+ /// If it is already contained, it's quietly ignored.+ pub fn insert(&mut self, id: CanonicalWordId) {+ self.inner.insert(id);+ }++ /// Is the list empty? In other words, Does this word have no known words it's derived from?+ pub fn is_empty(&self) -> bool {+ self.inner.is_empty()+ }++ /// Is the list empty? In other words, Does this word have no known words it's derived from?+ /// This is identical to `is_empty`. It's only here for backward-compatibility.+ pub fn is_none(&self) -> bool {+ self.is_empty()+ }++ /// Is this word derived from the word represented by `id`?+ pub fn contains(&self, id: CanonicalWordId) -> bool {+ self.inner.contains(&id)+ }++ /// Create a new `DerivedFrom` containing a single initial word ID.+ pub fn from_canonical_word_id(word_id: CanonicalWordId) -> Self {+ Self {+ inner: BTreeSet::from([word_id]),+ }+ }++ /// Get an iterator of the contained [`CanonicalWordId`].+ pub fn iter(&self) -> btree_set::Iter<'_, CanonicalWordId> {+ self.inner.iter()+ }+}++impl Extend<CanonicalWordId> for DerivedFrom {+ fn extend<T: IntoIterator<Item = CanonicalWordId>>(&mut self, iter: T) {+ self.inner.extend(iter);+ }+}++impl<'a> Extend<&'a CanonicalWordId> for DerivedFrom {+ fn extend<T: IntoIterator<Item = &'a CanonicalWordId>>(&mut self, iter: T) {+ self.inner.extend(iter);+ }+}++impl IntoIterator for DerivedFrom {+ type Item = CanonicalWordId;++ type IntoIter = btree_set::IntoIter<Self::Item>;++ fn into_iter(self) -> Self::IntoIter {+ self.inner.into_iter()+ }+}diff --git a/harper-core/src/dict_word_metadata_orthography.rs b/harper-core/src/dict_word_metadata_orthography.rsindex 3b0254c0b..0d6411bbf 100644--- a/harper-core/src/dict_word_metadata_orthography.rs+++ b/harper-core/src/dict_word_metadata_orthography.rs@@ -54,6 +54,18 @@ impl Default for OrthFlags {}impl OrthFlags {+ /// An `OrthFlags` with all casing-related flags set.+ pub const CASE_FLAGS: OrthFlags = OrthFlags::LOWERCASE+ .union(OrthFlags::TITLECASE)+ .union(OrthFlags::ALLCAPS)+ .union(OrthFlags::LOWER_CAMEL)+ .union(OrthFlags::UPPER_CAMEL);++ /// Mask out all flags not related to casing.+ pub const fn case_flags(self) -> Self {+ self.intersection(Self::CASE_FLAGS)+ }+/// Construct orthography flags for a given sequence of letters.pub fn from_letters(letters: &[char]) -> Self {let mut ortho_flags = Self::default();diff --git a/harper-core/src/linting/orthographic_consistency.rs b/harper-core/src/linting/orthographic_consistency.rsindex 48ea41cd7..bf85de99d 100644--- a/harper-core/src/linting/orthographic_consistency.rs+++ b/harper-core/src/linting/orthographic_consistency.rs@@ -66,12 +66,16 @@ impl ExprLinter for OrthographicConsistency {let chars = word.get_ch(source);+ if self.dict.contains_exact_word(chars) {+ // Exit if the dictionary contains the exact word.+ return None;+ }+let cur_flags = OrthFlags::from_letters(chars);if metadata.is_allcaps()- && !metadata.is_lowercase()- && !metadata.is_upper_camel()&& !cur_flags.contains(OrthFlags::ALLCAPS)+ && metadata.orth_info.case_flags().bits().count_ones() == 1{return Some(Lint {span: word.span,@@ -101,7 +105,7 @@ impl ExprLinter for OrthographicConsistency {&& !canonical_flags.contains(OrthFlags::TITLECASE)}).count()- == 1+ != 0&& let Some(canonical) = self.dict.get_correct_capitalization_of(chars)&& alphabetic_differs(canonical, chars){@@ -148,7 +152,10 @@ fn alphabetic_differs(a: &[char], b: &[char]) -> bool {#[cfg(test)]mod tests {- use crate::linting::tests::{assert_no_lints, assert_suggestion_result};+ use crate::linting::tests::{+ assert_good_and_bad_suggestions, assert_lint_count, assert_no_lints,+ assert_suggestion_result,+ };use super::OrthographicConsistency;@@ -170,15 +177,6 @@ mod tests {);}- #[test]- fn lego_should_be_all_caps() {- assert_suggestion_result(- "Lego bricks encourage creativity.",- OrthographicConsistency::default(),- "LEGO bricks encourage creativity.",- );- }-#[test]fn nato_should_be_all_caps() {assert_suggestion_result(@@ -453,6 +451,33 @@ mod tests {);}+ #[test]+ fn no_improper_suggestion_for_macos() {+ assert_good_and_bad_suggestions(+ "MacOS",+ OrthographicConsistency::default(),+ &["macOS"],+ &["MacOS"],+ );+ }++ #[test]+ fn accept_case_variants() {+ // At the time of writing this test, "Pr" (despite being a word in the curated dictionary)+ // would be linted for the supposed reason of the canonical spelling being "PR".+ // Since both words are in the curated dictionary, neither should be linted.+ assert_no_lints("Pr PR", OrthographicConsistency::default());+ assert_no_lints("Apr APR", OrthographicConsistency::default()); // https://github.com/Automattic/harper/issues/3080+ assert_no_lints("install.md", OrthographicConsistency::default()); // https://github.com/Automattic/harper/issues/3073+ }++ #[test]+ #[ignore = "Words with multiple known orthographic variants are not currently supported"]+ fn dont_accept_undefined_case_variants() {+ // "pr" isn't defined in the dictionary, so it should be linted.+ assert_lint_count("pr", OrthographicConsistency::default(), 1);+ }+#[test]fn lets_confusion_description() {assert_no_lints(diff --git a/harper-core/src/linting/sentence_capitalization.rs b/harper-core/src/linting/sentence_capitalization.rsindex b49d1ef68..a514118ac 100644--- a/harper-core/src/linting/sentence_capitalization.rs+++ b/harper-core/src/linting/sentence_capitalization.rs@@ -37,7 +37,7 @@ impl<T: Dictionary> Linter for SentenceCapitalization<T> {}}- for sentence in paragraph.iter_sentences() {+ 'sentence: for sentence in paragraph.iter_sentences() {if !is_full_sentence(sentence) {continue;}@@ -53,12 +53,12 @@ impl<T: Dictionary> Linter for SentenceCapitalization<T> {&& first_char.is_alphabetic()&& !first_char.is_uppercase(){- if let Some(canonical_spelling) =- self.dictionary.get_correct_capitalization_of(word_chars)+ for canonical_spelling in+ self.dictionary.get_correct_capitalizations_of(word_chars){// Skip if it's a proper noun or contains uppercase letters before a separatorif first_word.kind.is_proper_noun() {- continue;+ continue 'sentence;}// Check for uppercase letters in the rest of the word before any separators@@ -68,7 +68,7 @@ impl<T: Dictionary> Linter for SentenceCapitalization<T> {.take_while(|&c| !c.is_whitespace() && *c != '-' && *c != '\'').any(|&c| c.is_uppercase()){- continue;+ continue 'sentence;}}diff --git a/harper-core/src/linting/spell_check.rs b/harper-core/src/linting/spell_check.rsindex 792035657..3d9da5b97 100644--- a/harper-core/src/linting/spell_check.rs+++ b/harper-core/src/linting/spell_check.rs@@ -1,5 +1,6 @@use std::num::NonZero;+use itertools::Itertools;use lru::LruCache;use smallvec::ToSmallVec;@@ -53,6 +54,7 @@ impl<T: Dictionary> SpellCheck<T> {.dialects.is_dialect_enabled(self.dialect)})+ .unique_by(|v| v.to_lower()).map(|v| v.to_smallvec()).take(Self::MAX_SUGGESTIONS).collect();@@ -100,7 +102,12 @@ impl<T: Dictionary> Linter for SpellCheck<T> {if let Some(metadata) = word.kind.as_word().unwrap()&& metadata.dialects.is_dialect_enabled(self.dialect)&& (self.dictionary.contains_exact_word(word_chars)- || self.dictionary.contains_exact_word(&word_chars.to_lower()))+ || self+ .dictionary+ .contains_exact_word(word_chars.normalized().as_ref())+ || self+ .dictionary+ .contains_exact_word(word_chars.normalized().to_lower().as_ref())){continue;};@@ -169,7 +176,7 @@ mod tests {use crate::dict_word_metadata::DialectFlags;use crate::linting::Linter;use crate::linting::pooled_linter::for_tests::create_test_pool;- use crate::linting::tests::{assert_good_and_bad_suggestions, assert_no_lints};+ use crate::linting::tests::assert_no_lints;use crate::spell::{Dictionary, FstDictionary, MergedDictionary, MutableDictionary};use crate::{Dialect,@@ -527,18 +534,10 @@ mod tests {}#[test]- fn dont_flag_pr() {+ fn dont_flag_certain_entries_with_multiple_case_variants_in_dict() {assert_no_lints("PR", test_linter());- }-- #[test]- fn no_improper_suggestion_for_macos() {- assert_good_and_bad_suggestions("MacOS", test_linter(), &["macOS"], &["MacOS"]);- }-- #[test]- fn allows_parenthetical_plural_s() {- assert_no_lints("Please ask each person(s) to sign.", test_linter());+ assert_no_lints("MB", test_linter());+ assert_no_lints("OS", test_linter()); // Issue #2585}#[test]diff --git a/harper-core/src/linting/split_words.rs b/harper-core/src/linting/split_words.rsindex 4ee9e0b81..4a61579f8 100644--- a/harper-core/src/linting/split_words.rs+++ b/harper-core/src/linting/split_words.rs@@ -103,21 +103,21 @@ impl ExprLinter for SplitWords {let remainder = &chars[split_pos..];// Both parts must be valid common words- let Some(cand_meta) = self.dict.get_word_metadata(candidate) else {+ let Some(cand_meta) = self.dict.get_word_metadata_exact(candidate) else {continue;};if !cand_meta.common {continue;}- let Some(rem_meta) = self.dict.get_word_metadata(remainder) else {+ let Some(rem_meta) = self.dict.get_word_metadata_exact(remainder) else {continue;};if !rem_meta.common {continue;}- if is_anchor_split(&cand_meta) || is_anchor_split(&rem_meta) {+ if is_anchor_split(cand_meta) || is_anchor_split(rem_meta) {has_anchor_split = true;}diff --git a/harper-core/src/patterns/derived_from.rs b/harper-core/src/patterns/derived_from.rsindex bb1d0ac93..17878fcc8 100644--- a/harper-core/src/patterns/derived_from.rs+++ b/harper-core/src/patterns/derived_from.rs@@ -1,4 +1,4 @@-use crate::spell::WordId;+use crate::spell::{CanonicalWordId, CaseFoldedWordId};use super::Pattern;@@ -7,19 +7,19 @@ use super::Pattern;////// For example, this will match "call" as well as "recall", "calling", etc.pub struct DerivedFrom {- word_id: WordId,+ word_id: CanonicalWordId,}impl DerivedFrom {pub fn new_from_str(word: &str) -> DerivedFrom {- Self::new(WordId::from_word_str(word))+ Self::new(CanonicalWordId::from_word_str(word))}pub fn new_from_chars(word: &[char]) -> DerivedFrom {- Self::new(WordId::from_word_chars(word))+ Self::new(CanonicalWordId::from_word_chars(word))}- pub fn new(word_id: WordId) -> Self {+ pub fn new(word_id: CanonicalWordId) -> Self {Self { word_id }}}@@ -27,19 +27,21 @@ impl DerivedFrom {impl Pattern for DerivedFrom {fn matches(&self, tokens: &[crate::Token], source: &[char]) -> Option<usize> {let tok = tokens.first()?;- let metadata = tok.kind.as_word()?.as_ref()?;-- if metadata.derived_from == Some(self.word_id) {- return Some(1);- }-let chars = tok.get_ch(source);- let word_id = WordId::from_word_chars(chars);- if word_id == self.word_id {- return Some(1);+ let is_exact_match = CanonicalWordId::from_word_chars(chars) == self.word_id+ || CaseFoldedWordId::from_word_chars(chars).0 == self.word_id.as_case_folded();++ if is_exact_match+ || tok+ .kind+ .as_word()?+ .as_ref()+ .is_some_and(|meta| meta.derived_from.contains(self.word_id))+ {+ Some(1)+ } else {+ None}-- None}}diff --git a/harper-core/src/spell/dictionary.rs b/harper-core/src/spell/dictionary.rsindex f06a8f92b..d82c74b10 100644--- a/harper-core/src/spell/dictionary.rs+++ b/harper-core/src/spell/dictionary.rs@@ -1,10 +1,13 @@use blanket::blanket;use std::borrow::Cow;+use super::CanonicalWordId;use super::FuzzyMatchResult;-use super::WordId;use crate::DictWordMetadata;+pub static CURATED_DICT_STR: &str = include_str!("../../dictionary.dict");+pub static ANNOTATIONS_STR: &str = include_str!("../../annotations.json");+/// An in-memory database that contains everything necessary to parse and analyze English text.////// See also: [`super::FstDictionary`] and [`super::MutableDictionary`].@@ -32,7 +35,13 @@ pub trait Dictionary: Send + Sync {max_distance: u8,max_results: usize,) -> Vec<FuzzyMatchResult<'_>>;+ /// Gets the word's canonical capitalization, if exactly one exists.+ /// If there are multiple possibilities, `None` is returned.fn get_correct_capitalization_of(&self, word: &[char]) -> Option<&'_ [char]>;+ fn get_correct_capitalizations_of(+ &self,+ word: &[char],+ ) -> Box<dyn Iterator<Item = &[char]> + Send + '_>;/// Get the associated [`DictWordMetadata`] for any capitalization of a given word.fn get_word_metadata(&self, word: &[char]) -> Option<Cow<'_, DictWordMetadata>>;/// Get the associated [`DictWordMetadata`] for any capitalization of a given word.@@ -40,6 +49,8 @@ pub trait Dictionary: Send + Sync {/// empty.fn get_word_metadata_str(&self, word: &str) -> Option<Cow<'_, DictWordMetadata>>;+ fn get_word_metadata_exact(&self, word: &[char]) -> Option<&DictWordMetadata>;+/// Iterate over the words in the dictionary.fn words_iter(&self) -> Box<dyn Iterator<Item = &'_ [char]> + Send + '_>;@@ -47,7 +58,7 @@ pub trait Dictionary: Send + Sync {fn word_count(&self) -> usize;/// Returns the correct capitalization of the word with the given ID.- fn get_word_from_id(&self, id: &WordId) -> Option<&[char]>;+ fn get_word_from_id(&self, id: &CanonicalWordId) -> Option<&[char]>;/// Look for words with a specific prefixfn find_words_with_prefix(&self, prefix: &[char]) -> Vec<Cow<'_, [char]>>;diff --git a/harper-core/src/spell/fst_dictionary.rs b/harper-core/src/spell/fst_dictionary.rsindex e4d9cffa6..f28df671c 100644--- a/harper-core/src/spell/fst_dictionary.rs+++ b/harper-core/src/spell/fst_dictionary.rs@@ -8,7 +8,7 @@ use fst::{IntoStreamer, Map as FstMap, Streamer, map::StreamWithState};use hashbrown::HashMap;use levenshtein_automata::{DFA, LevenshteinAutomatonBuilder};-use super::{Dictionary, FuzzyMatchResult, MutableDictionary, WordId};+use super::{CanonicalWordId, Dictionary, FuzzyMatchResult, MutableDictionary};use crate::{CharString, CharStringExt, DictWordMetadata};/// An immutable dictionary allowing for very fast spellchecking.@@ -62,7 +62,7 @@ impl FstDictionary {for (word_chars, _) in words.iter() {let word = word_chars.iter().collect::<String>();builder- .insert(word, WordId::from_word_chars(word_chars).into())+ .insert(word, CanonicalWordId::from_word_chars(word_chars).into()).expect("Insertion not in lexicographical order!");}@@ -139,6 +139,17 @@ impl Dictionary for FstDictionary {self.mutable_dict.get_word_metadata_str(word)}+ fn get_word_metadata_exact(&self, word: &[char]) -> Option<&DictWordMetadata> {+ self.mutable_dict.get_word_metadata_exact(word)+ }++ fn get_correct_capitalizations_of(+ &self,+ word: &[char],+ ) -> Box<dyn Iterator<Item = &[char]> + Send + '_> {+ self.mutable_dict.get_correct_capitalizations_of(word)+ }+fn fuzzy_match(&'_ self,word: &[char],@@ -177,11 +188,11 @@ impl Dictionary for FstDictionary {let mut merged = Vec::with_capacity(best_distances.len());for (word_id, edit_distance) in best_distances {let word = self.mutable_dict.get_word_from_id(&word_id.into()).unwrap();- let metadata = self.mutable_dict.get_word_metadata(word).unwrap();+ let metadata = self.mutable_dict.get_word_metadata_exact(word).unwrap();merged.push(FuzzyMatchResult {word,edit_distance,- metadata,+ metadata: Cow::Borrowed(metadata),});}@@ -230,7 +241,7 @@ impl Dictionary for FstDictionary {self.mutable_dict.get_correct_capitalization_of(word)}- fn get_word_from_id(&self, id: &WordId) -> Option<&[char]> {+ fn get_word_from_id(&self, id: &CanonicalWordId) -> Option<&[char]> {self.mutable_dict.get_word_from_id(id)}@@ -249,7 +260,7 @@ mod tests {use crate::CharStringExt;use crate::DictWordMetadata;- use crate::spell::{Dictionary, MutableDictionary, WordId};+ use crate::spell::{CanonicalWordId, Dictionary, MutableDictionary};use super::FstDictionary;@@ -387,12 +398,11 @@ mod tests {fn plural_llamas_derived_from_llama() {let dict = FstDictionary::curated();- assert_eq!(+ assert!(dict.get_word_metadata_str("llamas").unwrap().derived_from- .unwrap(),- WordId::from_word_str("llama")+ .contains(CanonicalWordId::from_word_str("llama")))}@@ -400,12 +410,11 @@ mod tests {fn plural_cats_derived_from_cat() {let dict = FstDictionary::curated();- assert_eq!(+ assert!(dict.get_word_metadata_str("cats").unwrap().derived_from- .unwrap(),- WordId::from_word_str("cat")+ .contains(CanonicalWordId::from_word_str("cat")));}@@ -413,12 +422,11 @@ mod tests {fn unhappy_derived_from_happy() {let dict = FstDictionary::curated();- assert_eq!(+ assert!(dict.get_word_metadata_str("unhappy").unwrap().derived_from- .unwrap(),- WordId::from_word_str("happy")+ .contains(CanonicalWordId::from_word_str("happy")));}@@ -426,12 +434,11 @@ mod tests {fn quickly_derived_from_quick() {let dict = FstDictionary::curated();- assert_eq!(+ assert!(dict.get_word_metadata_str("quickly").unwrap().derived_from- .unwrap(),- WordId::from_word_str("quick")+ .contains(CanonicalWordId::from_word_str("quick")));}diff --git a/harper-core/src/spell/merged_dictionary.rs b/harper-core/src/spell/merged_dictionary.rsindex 7e5e731d9..b3e3a56a8 100644--- a/harper-core/src/spell/merged_dictionary.rs+++ b/harper-core/src/spell/merged_dictionary.rs@@ -5,7 +5,7 @@ use std::sync::Arc;use foldhash::quality::FixedState;use itertools::Itertools;-use super::{FstDictionary, FuzzyMatchResult, WordId, dictionary::Dictionary};+use super::{CanonicalWordId, FstDictionary, FuzzyMatchResult, dictionary::Dictionary};use crate::{CharString, DictWordMetadata};/// A simple wrapper over [`Dictionary`] that allows@@ -136,6 +136,24 @@ impl Dictionary for MergedDictionary {self.get_word_metadata(&chars)}+ fn get_word_metadata_exact(&self, word: &[char]) -> Option<&DictWordMetadata> {+ self.children+ .iter()+ .find_map(|child| child.get_word_metadata_exact(word))+ }++ fn get_correct_capitalizations_of(+ &self,+ word: &[char],+ ) -> Box<dyn Iterator<Item = &[char]> + Send + '_> {+ let matches: Vec<_> = self+ .children+ .iter()+ .map(|child| child.get_correct_capitalizations_of(word))+ .collect();+ Box::new(matches.into_iter().flatten())+ }+fn fuzzy_match(&'_ self,word: &[char],@@ -172,7 +190,7 @@ impl Dictionary for MergedDictionary {self.children.iter().map(|d| d.word_count()).sum()}- fn get_word_from_id(&self, id: &WordId) -> Option<&[char]> {+ fn get_word_from_id(&self, id: &CanonicalWordId) -> Option<&[char]> {self.children.iter().find_map(|dict| dict.get_word_from_id(id))diff --git a/harper-core/src/spell/mod.rs b/harper-core/src/spell/mod.rsindex e4631e933..96ba0f26b 100644--- a/harper-core/src/spell/mod.rs+++ b/harper-core/src/spell/mod.rs@@ -10,7 +10,7 @@ pub use self::fst_dictionary::FstDictionary;pub use self::merged_dictionary::MergedDictionary;pub use self::mutable_dictionary::MutableDictionary;pub use self::trie_dictionary::TrieDictionary;-pub use self::word_id::WordId;+pub use self::word_id::{CanonicalWordId, CaseFoldedWordId};mod dictionary;mod fst_dictionary;diff --git a/harper-core/src/spell/mutable_dictionary.rs b/harper-core/src/spell/mutable_dictionary.rsindex 32a35e024..0d4bc9263 100644--- a/harper-core/src/spell/mutable_dictionary.rs+++ b/harper-core/src/spell/mutable_dictionary.rs@@ -1,7 +1,5 @@use super::{- FstDictionary, WordId,- rune::{self, AttributeList, parse_word_list},- word_map::{WordMap, WordMapEntry},+ CanonicalWordId, CaseFoldedWordId, FstDictionary, FuzzyMatchResult, rune, word_map::WordMap,};use crate::edit_distance::edit_distance_min_alloc;use itertools::Itertools;@@ -10,7 +8,6 @@ use std::{borrow::Cow, sync::LazyLock};use crate::{CharString, CharStringExt, DictWordMetadata};-use super::FuzzyMatchResult;use super::dictionary::Dictionary;/// A basic dictionary that allows words to be added after instantiating.@@ -31,8 +28,8 @@ pub struct MutableDictionary {/// curated dictionary.fn uncached_inner_new() -> Arc<MutableDictionary> {MutableDictionary::from_rune_files(- include_str!("../../dictionary.dict"),- include_str!("../../annotations.json"),+ super::dictionary::CURATED_DICT_STR,+ super::dictionary::ANNOTATIONS_STR,).map(Arc::new).unwrap_or_else(|e| panic!("Failed to load curated dictionary: {}", e))@@ -48,15 +45,9 @@ impl MutableDictionary {}pub fn from_rune_files(word_list: &str, attr_list: &str) -> Result<Self, rune::Error> {- let word_list = parse_word_list(word_list)?;- let attr_list = AttributeList::parse(attr_list)?;-- // There will be at _least_ this number of words- let mut word_map = WordMap::default();-- attr_list.expand_annotated_words(word_list, &mut word_map);-- Ok(Self { word_map })+ Ok(Self {+ word_map: WordMap::from_rune_files(word_list, attr_list)?,+ })}/// Create a dictionary from the curated dictionary included@@ -73,12 +64,7 @@ impl MutableDictionary {&mut self,words: impl IntoIterator<Item = (impl AsRef<[char]>, DictWordMetadata)>,) {- for (chars, metadata) in words.into_iter() {- self.word_map.insert(WordMapEntry {- metadata,- canonical_spelling: chars.as_ref().into(),- })- }+ self.word_map.extend_words(words);}/// Append a single word to the dictionary.@@ -106,13 +92,24 @@ impl Default for MutableDictionary {impl Dictionary for MutableDictionary {fn get_word_metadata(&self, word: &[char]) -> Option<Cow<'_, DictWordMetadata>> {- self.word_map- .get_with_chars(word)- .map(|v| Cow::Borrowed(&v.metadata))+ let mut found_words = self.word_map.get_case_folded_chars(word);+ let first = &found_words.next()?.metadata;++ if let Some(second) = found_words.next() {+ let mut first = first.clone();+ first.merge(&second.metadata);+ found_words.for_each(|entry| {+ first.merge(&entry.metadata);+ });+ Some(Cow::Owned(first))+ } else {+ Some(Cow::Borrowed(first))+ }}fn contains_word(&self, word: &[char]) -> bool {- self.word_map.contains_chars(word)+ self.word_map+ .contains_case_folded(CaseFoldedWordId::from_word_chars(word).0)}fn contains_word_str(&self, word: &str) -> bool {@@ -126,15 +123,32 @@ impl Dictionary for MutableDictionary {}fn get_correct_capitalization_of(&self, word: &[char]) -> Option<&'_ [char]> {+ let mut entries = self.word_map.get_case_folded_chars(word);+ let first = entries.next()?;+ if entries.next().is_some() {+ None+ } else {+ Some(first.canonical_spelling.as_slice())+ }+ }++ fn get_correct_capitalizations_of(+ &self,+ word: &[char],+ ) -> Box<dyn Iterator<Item = &[char]> + Send + '_> {+ Box::new(+ self.word_map+ .get_case_folded_chars(word)+ .map(|entry| entry.canonical_spelling.as_slice()),+ )+ }++ fn get_word_metadata_exact(&self, word: &[char]) -> Option<&DictWordMetadata> {self.word_map- .get_with_chars(word)- .map(|v| v.canonical_spelling.as_slice())+ .get_canonical(CanonicalWordId::from_word_chars(word))+ .map(|entry| &entry.metadata)}- /// Suggest a correct spelling for a given misspelled word.- /// `Self::word` is assumed to be quite small (n < 100).- /// `max_distance` relates to an optimization that allows the search- /// algorithm to prune large portions of the search.fn fuzzy_match(&'_ self,word: &[char],@@ -185,7 +199,7 @@ impl Dictionary for MutableDictionary {.map(|(word, edit_distance)| FuzzyMatchResult {word,edit_distance,- metadata: self.get_word_metadata(word).unwrap(),+ metadata: Cow::Borrowed(self.get_word_metadata_exact(word).unwrap()),}).collect()}@@ -213,15 +227,8 @@ impl Dictionary for MutableDictionary {}fn contains_exact_word(&self, word: &[char]) -> bool {- let normalized = word.normalized();-- if let Some(found) = self.word_map.get_with_chars(normalized.as_ref())- && found.canonical_spelling.as_ref() == normalized.as_ref()- {- return true;- }-- false+ self.word_map+ .contains_canonical(CanonicalWordId::from_word_chars(word))}fn contains_exact_word_str(&self, word: &str) -> bool {@@ -229,8 +236,10 @@ impl Dictionary for MutableDictionary {self.contains_exact_word(word.as_ref())}- fn get_word_from_id(&self, id: &WordId) -> Option<&[char]> {- self.word_map.get(id).map(|w| w.canonical_spelling.as_ref())+ fn get_word_from_id(&self, id: &CanonicalWordId) -> Option<&[char]> {+ self.word_map+ .get_canonical(*id)+ .map(|w| w.canonical_spelling.as_ref())}fn find_words_with_prefix(&self, prefix: &[char]) -> Vec<Cow<'_, [char]>> {diff --git a/harper-core/src/spell/rune/attribute_list.rs b/harper-core/src/spell/rune/attribute_list.rsindex 1dcb85ea5..52879ba67 100644--- a/harper-core/src/spell/rune/attribute_list.rs+++ b/harper-core/src/spell/rune/attribute_list.rs@@ -13,7 +13,7 @@ use super::expansion::{};use super::word_list::AnnotatedWord;use crate::dict_word_metadata_orthography::OrthFlags;-use crate::spell::WordId;+use crate::spell::CanonicalWordId;use crate::{CharString, DictWordMetadata, Span};#[derive(Debug, Clone)]@@ -64,6 +64,8 @@ impl AttributeList {let orth_flags = OrthFlags::from_letters(&annotated_word.letters);base_metadata.orth_info = orth_flags;+ let annotated_word_id = CanonicalWordId::from_word_chars(&annotated_word.letters);+let mut conditional_expansion_metadata = Vec::new();// First pass: Process all properties to build the base metadata@@ -149,25 +151,25 @@ impl AttributeList {word_map,);// Update the metadata of the expanded word- let target_metadata = word_map.get_metadata_mut_chars(&new_word).unwrap();+ let target_metadata = word_map+ .get_metadata_mut_canonical(CanonicalWordId::from_word_chars(new_word))+ .unwrap();target_metadata.merge(&metadata);- target_metadata.derived_from =- Some(WordId::from_word_chars(&annotated_word.letters));+ target_metadata.derived_from.insert(annotated_word_id);}} else {// Simple case: no cross-product expansion neededfor (key, mut value) in new_words.into_iter() {- value.derived_from = Some(WordId::from_word_chars(&annotated_word.letters));+ value.derived_from.insert(annotated_word_id);- if let Some(existing_metadata) = word_map.get_metadata_mut_chars(&key) {+ if let Some(existing_metadata) =+ word_map.get_metadata_mut_canonical(CanonicalWordId::from_word_chars(&key))+ {// Merge with existing metadataexisting_metadata.merge(&value);} else {// Add new entry- word_map.insert(WordMapEntry {- canonical_spelling: key,- metadata: value,- });+ word_map.insert(WordMapEntry::new(key).with_md(value));}}}@@ -177,15 +179,12 @@ impl AttributeList {let mut full_metadata = base_metadata;// Merge with any existing metadata for this word- if let Some(existing_metadata) = word_map.get_with_chars(&annotated_word.letters) {- full_metadata.merge(&existing_metadata.metadata);+ if let Some(existing_entry) = word_map.get_canonical(annotated_word_id) {+ full_metadata.merge(&existing_entry.metadata);}// Store the final metadata for the base word- word_map.insert(WordMapEntry {- metadata: full_metadata.clone(),- canonical_spelling: annotated_word.letters,- });+ word_map.insert(WordMapEntry::new(annotated_word.letters).with_md(full_metadata.clone()));// Process any conditional expansionsfor (letters, metadata, condition) in conditional_expansion_metadata {@@ -197,7 +196,7 @@ impl AttributeList {// Apply the conditional metadataword_map- .get_metadata_mut_chars(&letters)+ .get_metadata_mut_canonical(CanonicalWordId::from_word_chars(letters)).unwrap().merge(&metadata);}diff --git a/harper-core/src/spell/rune/mod.rs b/harper-core/src/spell/rune/mod.rsindex 3dcdaa258..af13e5dd7 100644--- a/harper-core/src/spell/rune/mod.rs+++ b/harper-core/src/spell/rune/mod.rs@@ -19,6 +19,7 @@ mod tests {use super::super::word_map::WordMap;use super::word_list::parse_word_list;use crate::CharStringExt;+ use crate::spell::CanonicalWordId;use crate::spell::rune::AttributeList;pub const TEST_WORD_LIST: &str = "4\nhello\ntry/B\nwork/AB\nblank/";@@ -208,10 +209,14 @@ mod tests {attributes.expand_annotated_words(words, &mut expanded);- let giant_data = expanded.get_with_str("giant").unwrap();+ let giant_data = expanded+ .get_canonical(CanonicalWordId::from_word_str("giant"))+ .unwrap();assert!(giant_data.metadata.is_noun());- let giants_data = expanded.get_with_str("giants").unwrap();+ let giants_data = expanded+ .get_canonical(CanonicalWordId::from_word_str("giants"))+ .unwrap();assert!(giants_data.metadata.is_plural_noun());}}diff --git a/harper-core/src/spell/trie_dictionary.rs b/harper-core/src/spell/trie_dictionary.rsindex a8b582db7..660ab8814 100644--- a/harper-core/src/spell/trie_dictionary.rs+++ b/harper-core/src/spell/trie_dictionary.rs@@ -6,7 +6,7 @@ use trie_rs::iter::{Keys, PrefixIter, SearchIter};use crate::DictWordMetadata;-use super::{Dictionary, FstDictionary, FuzzyMatchResult, WordId};+use super::{CanonicalWordId, Dictionary, FstDictionary, FuzzyMatchResult};/// A [`Dictionary`] optimized for pre- and postfix search./// Wraps another dictionary to implement other operations.@@ -89,10 +89,21 @@ impl<D: Dictionary> Dictionary for TrieDictionary<D> {self.inner.word_count()}- fn get_word_from_id(&self, id: &WordId) -> Option<&[char]> {+ fn get_word_from_id(&self, id: &CanonicalWordId) -> Option<&[char]> {self.inner.get_word_from_id(id)}+ fn get_word_metadata_exact(&self, word: &[char]) -> Option<&DictWordMetadata> {+ self.inner.get_word_metadata_exact(word)+ }++ fn get_correct_capitalizations_of(+ &self,+ word: &[char],+ ) -> Box<dyn Iterator<Item = &[char]> + Send + '_> {+ self.inner.get_correct_capitalizations_of(word)+ }+fn find_words_with_prefix(&self, prefix: &[char]) -> Vec<Cow<'_, [char]>> {let results: Keys<SearchIter<'_, char, (), Vec<char>, _>> =self.trie.predictive_search(prefix);diff --git a/harper-core/src/spell/word_id.rs b/harper-core/src/spell/word_id.rsdeleted file mode 100644index f93fc2087..000000000--- a/harper-core/src/spell/word_id.rs+++ /dev/null@@ -1,46 +0,0 @@-use std::hash::BuildHasher;--use foldhash::fast::FixedState;-use serde::{Deserialize, Serialize};--use crate::{CharString, CharStringExt};--/// An identifier for a particular word.-///-/// It works by hashing the word it represents, normalized to lowercase.-/// It is meant for situations where you need to refer to a word (or a collection of words),-/// without storing all of accompanying data (like spelling or metadata).-#[derive(Hash, Copy, Clone, PartialEq, Eq, PartialOrd, Debug, Serialize, Deserialize)]-pub struct WordId {- hash: u64,-}--impl WordId {- /// Create a Word ID from a character slice.- pub fn from_word_chars(chars: impl AsRef<[char]>) -> Self {- let normalized = chars.as_ref().normalized();- let lower = normalized.to_lower();- let hash = FixedState::default().hash_one(lower);-- Self { hash }- }-- /// Create a word ID from a string.- /// Requires allocation, so use sparingly.- pub fn from_word_str(text: impl AsRef<str>) -> Self {- let chars: CharString = text.as_ref().chars().collect();- Self::from_word_chars(chars)- }-}--impl From<WordId> for u64 {- fn from(val: WordId) -> Self {- val.hash- }-}--impl From<u64> for WordId {- fn from(value: u64) -> Self {- Self { hash: value }- }-}diff --git a/harper-core/src/spell/word_id/canonical_word_id.rs b/harper-core/src/spell/word_id/canonical_word_id.rsnew file mode 100644index 000000000..33eef03b4--- /dev/null+++ b/harper-core/src/spell/word_id/canonical_word_id.rs@@ -0,0 +1,51 @@+use std::hash::BuildHasher;++use foldhash::fast::FixedState;+use serde::{Deserialize, Serialize};++use super::CaseFoldedWordId;+use crate::CharString;++/// An identifier for a particular word with canonical casing.+#[derive(Hash, Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Debug, Serialize, Deserialize)]+pub struct CanonicalWordId {+ pub hash: u64,+}++impl CanonicalWordId {+ /// Create a Word ID from a character slice.+ pub fn from_word_chars(chars: impl AsRef<[char]>) -> Self {+ let hash = FixedState::default().hash_one(chars.as_ref());++ Self { hash }+ }++ /// Create a word ID from a string.+ /// Requires allocation, so use sparingly.+ pub fn from_word_str(text: impl AsRef<str>) -> Self {+ let chars: CharString = text.as_ref().chars().collect();+ Self::from_word_chars(chars)+ }++ /// Reinterpret this ID as a [`CaseFoldedWordId`].+ ///+ /// Note that this is just a reinterpretation, it does not perform any conversion. This is+ /// useful when the canonical word ID is the same as the case-folded word ID. This will only+ /// happen if the canonical ID was generated with a word that was already lowercased and+ /// normalized.+ pub fn as_case_folded(self) -> CaseFoldedWordId {+ CaseFoldedWordId { hash: self.hash }+ }+}++impl From<CanonicalWordId> for u64 {+ fn from(val: CanonicalWordId) -> Self {+ val.hash+ }+}++impl From<u64> for CanonicalWordId {+ fn from(value: u64) -> Self {+ Self { hash: value }+ }+}diff --git a/harper-core/src/spell/word_id/case_folded_word_id.rs b/harper-core/src/spell/word_id/case_folded_word_id.rsnew file mode 100644index 000000000..393c1d8be--- /dev/null+++ b/harper-core/src/spell/word_id/case_folded_word_id.rs@@ -0,0 +1,64 @@+use std::{borrow::Cow, hash::BuildHasher};++use foldhash::fast::FixedState;+use serde::{Deserialize, Serialize};++use crate::{CharString, CharStringExt, spell::CanonicalWordId};++/// An identifier for a particular word with case-folded casing.+///+/// This does not usually point to a specific word, but rather a group of words that are identical+/// when lowercased.+#[derive(Hash, Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Debug, Serialize, Deserialize)]+pub struct CaseFoldedWordId {+ pub hash: u64,+}++impl CaseFoldedWordId {+ /// Create a Word ID from a character slice.+ ///+ /// This will Case-fold and normalize the input before calculating the word ID.+ ///+ /// If the input word was already case-folded, this will also return+ /// a [`CanonicalWordId`]. This is because the IDs will be identical under the hood, and the+ /// canonical ID can be produced with no additional cost.+ pub fn from_word_chars(chars: impl AsRef<[char]>) -> (Self, Option<CanonicalWordId>) {+ let normalized = chars.as_ref().normalized();+ let lower = normalized.to_lower();++ let was_already_case_folded =+ matches!(normalized, Cow::Borrowed(_)) && matches!(lower, Cow::Borrowed(_));++ let hash = FixedState::default().hash_one(lower);++ (+ Self { hash },+ was_already_case_folded.then_some(CanonicalWordId { hash }),+ )+ }++ /// Create a word ID from a string.+ /// Requires allocation, so use sparingly.+ ///+ /// This will case-fold and normalize the input before calculating the word ID.+ ///+ /// If the input word was already case-folded, this will also return+ /// a [`CanonicalWordId`]. This is because the IDs will be identical under the hood, and the+ /// canonical ID can be produced with no additional cost.+ pub fn from_word_str(text: impl AsRef<str>) -> (Self, Option<CanonicalWordId>) {+ let chars: CharString = text.as_ref().chars().collect();+ Self::from_word_chars(chars)+ }+}++impl From<CaseFoldedWordId> for u64 {+ fn from(val: CaseFoldedWordId) -> Self {+ val.hash+ }+}++impl From<u64> for CaseFoldedWordId {+ fn from(value: u64) -> Self {+ Self { hash: value }+ }+}diff --git a/harper-core/src/spell/word_id/mod.rs b/harper-core/src/spell/word_id/mod.rsnew file mode 100644index 000000000..37f8ed4b2--- /dev/null+++ b/harper-core/src/spell/word_id/mod.rs@@ -0,0 +1,12 @@+//! Identifiers for words.+//!+//! These are meant for situations where you need to refer to a word (or a collection of words),+//! without storing all of accompanying data (like spelling or metadata).++pub use canonical_word_id::CanonicalWordId;+pub use case_folded_word_id::CaseFoldedWordId;+pub use word_id_pair::WordIdPair;++mod canonical_word_id;+mod case_folded_word_id;+mod word_id_pair;diff --git a/harper-core/src/spell/word_id/word_id_pair.rs b/harper-core/src/spell/word_id/word_id_pair.rsnew file mode 100644index 000000000..523ed2742--- /dev/null+++ b/harper-core/src/spell/word_id/word_id_pair.rs@@ -0,0 +1,45 @@+use serde::{Deserialize, Serialize};++use super::{CanonicalWordId, CaseFoldedWordId};+use crate::CharString;++/// A pair containing both [`CanonicalWordId`] and [`CaseFoldedWordId`] for a given word.+#[derive(Hash, Copy, Clone, PartialEq, Eq, PartialOrd, Debug, Serialize, Deserialize)]+pub struct WordIdPair {+ pub canonical: CanonicalWordId,+ pub case_folded: CaseFoldedWordId,+}++impl WordIdPair {+ /// Create a Word ID pair from a character slice.+ ///+ /// Calculates both the canonical and case-folded word ID for the provided word.+ pub fn from_word_chars(chars: impl AsRef<[char]>) -> Self {+ let (case_folded, maybe_canonical) = CaseFoldedWordId::from_word_chars(chars.as_ref());++ Self {+ // Trying to avoid hashing again if possible.+ canonical: maybe_canonical.unwrap_or_else(|| CanonicalWordId::from_word_chars(chars)),+ case_folded,+ }+ }++ /// Create a word ID pair from a string.+ /// Requires allocation, so use sparingly.+ ///+ /// Calculates both the canonical and case-folded word ID for the provided word.+ pub fn from_word_str(text: impl AsRef<str>) -> Self {+ let chars: CharString = text.as_ref().chars().collect();+ Self::from_word_chars(chars)+ }++ /// The canonical ID of the word.+ pub fn canonical(&self) -> CanonicalWordId {+ self.canonical+ }++ /// The case-folded ID of the word.+ pub fn case_folded(&self) -> CaseFoldedWordId {+ self.case_folded+ }+}diff --git a/harper-core/src/spell/word_map.rs b/harper-core/src/spell/word_map.rsdeleted file mode 100644index 8f97ec884..000000000--- a/harper-core/src/spell/word_map.rs+++ /dev/null@@ -1,104 +0,0 @@-use hashbrown::{HashMap, hash_map::IntoValues};--use crate::{CharString, DictWordMetadata};--use super::WordId;--/// The underlying data structure for the `MutableDictionary`.-#[derive(Debug, Clone, Eq, PartialEq, Default)]-pub struct WordMap {- inner: HashMap<WordId, WordMapEntry>,-}--#[derive(Debug, Clone, Eq, PartialEq, Hash)]-pub struct WordMapEntry {- pub metadata: DictWordMetadata,- pub canonical_spelling: CharString,-}--impl WordMap {- /// Get an entry from the word map using raw chars.- pub fn get_with_str(&self, string: &str) -> Option<&WordMapEntry> {- let chars: CharString = string.chars().collect();- let id = WordId::from_word_chars(chars);-- self.get(&id)- }-- pub fn contains_str(&self, string: &str) -> bool {- self.get_with_str(string).is_some()- }-- pub fn contains_chars(&self, chars: impl AsRef<[char]>) -> bool {- self.get_with_chars(chars).is_some()- }-- pub fn contains(&self, id: &WordId) -> bool {- self.get(id).is_some()- }-- /// Get an entry from the word map using raw chars.- pub fn get_with_chars(&self, chars: impl AsRef<[char]>) -> Option<&WordMapEntry> {- let id = WordId::from_word_chars(chars);-- self.get(&id)- }-- /// Get an entry from the word map using a word identifier.- pub fn get(&self, id: &WordId) -> Option<&WordMapEntry> {- self.inner.get(id)- }-- /// Borrow a word's metadata mutably- pub fn get_metadata_mut_chars(- &mut self,- chars: impl AsRef<[char]>,- ) -> Option<&mut DictWordMetadata> {- let id = WordId::from_word_chars(chars);-- self.get_metadata_mut(&id)- }-- /// Borrow a word's metadata mutably- pub fn get_metadata_mut(&mut self, id: &WordId) -> Option<&mut DictWordMetadata> {- self.inner.get_mut(id).map(|v| &mut v.metadata)- }-- pub fn insert(&mut self, entry: WordMapEntry) {- let id = WordId::from_word_chars(&entry.canonical_spelling);-- self.inner.insert(id, entry);- }-- /// Reserves capacity for at least `additional` more elements to be inserted- /// in the `WordMap`. The collection may reserve more space to avoid- /// frequent reallocations.- pub fn reserve(&mut self, additional: usize) {- self.inner.reserve(additional);- }-- /// Iterate through the canonical spellings of the words in the map.- pub fn iter(&self) -> impl Iterator<Item = &WordMapEntry> {- self.inner.values()- }-- pub fn len(&self) -> usize {- self.inner.len()- }-- pub fn with_capacity(capacity: usize) -> Self {- Self {- inner: HashMap::with_capacity(capacity),- }- }-}--impl IntoIterator for WordMap {- type Item = WordMapEntry;-- fn into_iter(self) -> Self::IntoIter {- self.inner.into_values()- }-- type IntoIter = IntoValues<WordId, WordMapEntry>;-}diff --git a/harper-core/src/spell/word_map/mod.rs b/harper-core/src/spell/word_map/mod.rsnew file mode 100644index 000000000..e080dd643--- /dev/null+++ b/harper-core/src/spell/word_map/mod.rs@@ -0,0 +1,252 @@+use std::{+ ops::Index,+ sync::{Arc, LazyLock},+};++use hashbrown::{DefaultHashBuilder, HashMap};+use indexmap::IndexMap;++use crate::{+ DictWordMetadata,+ spell::{+ dictionary::{ANNOTATIONS_STR, CURATED_DICT_STR},+ rune::{self, AttributeList, parse_word_list},+ word_id::{CanonicalWordId, CaseFoldedWordId},+ },+};++pub mod word_map_entry;+pub use word_map_entry::WordMapEntry;++/// Type used to store the canonical entries in the word map.+///+/// This currently uses an [`IndexMap`], so a word can be indexed either by its canonical word ID+/// (which is the same across all word maps), or directly via a `usize` index (specific to this+/// word map only).+///+/// The latter property is used to allow [`WordMap::case_folded`] to indirectly reference entries+/// in [`WordMap::canonical`].+type CanonicalStorage = IndexMap<CanonicalWordId, WordMapEntry, DefaultHashBuilder>;++/// The most basic dictionary. Stores a list of words and their metadata, while allowing for+/// efficient case-folded lookups.+///+/// This is the underlying data structure for all other dictionaries (at the time of writing).+#[derive(Debug, Clone, Eq, PartialEq, Default)]+pub struct WordMap {+ /// Underlying container for the entries in the word map.+ canonical: CanonicalStorage,+ /// A map containing indices into `canonical` for a specific `CaseFoldedWordId`. This is used for+ /// case-folded lookups in the word map.+ case_folded: HashMap<CaseFoldedWordId, Vec<usize>>,+}++impl WordMap {+ /// Create an empty word map.+ pub fn new() -> Self {+ Default::default()+ }++ /// A word map containing entries from the curated dictionary.+ pub fn curated() -> Arc<Self> {+ /// A word map containing entries from the curated dictionary.+ static CURATED: LazyLock<Arc<WordMap>> = LazyLock::new(|| {+ Arc::new(WordMap::from_rune_files(CURATED_DICT_STR, ANNOTATIONS_STR).unwrap())+ });++ (*CURATED).clone()+ }++ /// Does the word map contain a word with the given [`CanonicalWordId`]?+ pub fn contains_canonical(&self, id: CanonicalWordId) -> bool {+ self.get_canonical(id).is_some()+ }++ /// Does the word map contain any words with the given [`CaseFoldedWordId`]?+ pub fn contains_case_folded(&self, id: CaseFoldedWordId) -> bool {+ !self.get_canonical_indices_from_case_folded(id).is_empty()+ }++ /// Get an entry from the word map using a word identifier.+ pub fn get_canonical(&self, id: CanonicalWordId) -> Option<&WordMapEntry> {+ self.canonical.get(&id)+ }++ /// Get the entries corresponding to the provided [`CaseFoldedWordId`].+ pub fn get_case_folded(+ &self,+ id: CaseFoldedWordId,+ ) -> impl ExactSizeIterator<Item = &WordMapEntry> {+ self.get_canonical_indices_from_case_folded(id)+ .iter()+ .map(|canonical_index| &self[*canonical_index])+ }++ /// Convenience wrapper for [`Self::get_case_folded`].+ ///+ /// This recalculates the word ID every time. If you're going to be querying the same word+ /// multiple times, consider storing the word ID and using [`Self::get_case_folded`] instead.+ pub fn get_case_folded_chars(+ &self,+ word: &[char],+ ) -> impl ExactSizeIterator<Item = &WordMapEntry> + use<'_> {+ self.get_case_folded(CaseFoldedWordId::from_word_chars(word).0)+ }++ /// Borrow a word's metadata mutably+ pub fn get_metadata_mut_canonical(+ &mut self,+ id: CanonicalWordId,+ ) -> Option<&mut DictWordMetadata> {+ self.canonical.get_mut(&id).map(|v| &mut v.metadata)+ }++ /// Insert an entry into the word map.+ ///+ /// This will merge the new metadata with the existing if an entry with an identical+ /// [`CanonicalWordId`] is already contained in the word map.+ ///+ /// If you are appending many words, consider using [`Self::extend`] instead.+ pub fn insert(&mut self, entry: WordMapEntry) {+ let (canonical_id, case_folded_id) =+ (entry.word_ids.canonical(), entry.word_ids.case_folded());++ if let Some(existing_entry) = self.canonical.get_mut(&canonical_id) {+ // An existing word with the same canonical ID exists; update its entry.+ existing_entry.metadata.merge(&entry.metadata);+ } else {+ // An existing word with the same canonical ID does NOT exist; insert it.+ let (canonical_idx, _) = self.canonical.insert_full(canonical_id, entry);+ if let Some(existing_case_folded_entry) = self.case_folded.get_mut(&case_folded_id) {+ // `case_folded` already has a canonical ID list for this word; append to it, if+ // the same entry does not already exist.+ if !existing_case_folded_entry.contains(&canonical_idx) {+ existing_case_folded_entry.push(canonical_idx);+ }+ } else {+ // `case_folded` does NOT have a canonical ID list for this word; initialize one.+ self.case_folded.insert(case_folded_id, vec![canonical_idx]);+ }+ }+ }++ /// Reserves capacity for at least `additional` more elements to be inserted+ /// in the `WordMap`. The collection may reserve more space to avoid+ /// frequent reallocations.+ pub fn reserve(&mut self, additional: usize) {+ self.canonical.reserve(additional);+ }++ /// Iterate through the canonical spellings of the words in the map.+ pub fn iter(&self) -> impl ExactSizeIterator<Item = &WordMapEntry> {+ self.canonical.values()+ }++ /// The number of words in the word map.+ pub fn len(&self) -> usize {+ self.canonical.len()+ }++ pub fn is_empty(&self) -> bool {+ self.len() == 0+ }++ pub fn with_capacity(capacity: usize) -> Self {+ Self {+ canonical: IndexMap::with_capacity_and_hasher(capacity, DefaultHashBuilder::default()),+ case_folded: HashMap::new(),+ }+ }++ pub fn from_rune_files(word_list: &str, attr_list: &str) -> Result<Self, rune::Error> {+ let word_list = parse_word_list(word_list)?;+ let attr_list = AttributeList::parse(attr_list)?;++ let mut word_map = WordMap::default();++ attr_list.expand_annotated_words(word_list, &mut word_map);++ Ok(word_map)+ }++ /// Get indices into [`Self::canonical`] using the provided [`CaseFoldedWordId`].+ fn get_canonical_indices_from_case_folded(&self, id: CaseFoldedWordId) -> &[usize] {+ self.case_folded+ .get(&id)+ .map_or(&[], |canonical_indices| canonical_indices)+ }+}++// Functions for backward compatibility.+impl WordMap {+ /// Append a single word to the dictionary.+ ///+ /// If you are appending many words, consider using [`Self::extend_words`]+ /// instead.+ pub fn append_word(&mut self, word: impl AsRef<[char]>, metadata: DictWordMetadata) {+ self.insert(WordMapEntry::new(word.as_ref()).with_md(metadata));+ }++ /// Append a single word to the dictionary.+ ///+ /// If you are appending many words, consider using [`Self::extend_words`]+ /// instead.+ pub fn append_word_str(&mut self, word: &str, metadata: DictWordMetadata) {+ self.insert(WordMapEntry::new_str(word).with_md(metadata));+ }++ /// Appends words to the dictionary.+ pub fn extend_words(+ &mut self,+ words: impl IntoIterator<Item = (impl AsRef<[char]>, DictWordMetadata)>,+ ) {+ self.extend(words.into_iter().map(|(canonical_spelling, metadata)| {+ WordMapEntry::new(canonical_spelling.as_ref()).with_md(metadata)+ }))+ }+}++impl Extend<WordMapEntry> for WordMap {+ fn extend<T: IntoIterator<Item = WordMapEntry>>(&mut self, iter: T) {+ // [Copied from HashBrown::map::HashMap::extend]+ // Keys may be already present or show multiple times in the iterator.+ // Reserve the entire hint lower bound if the map is empty.+ // Otherwise reserve half the hint (rounded up), so the map+ // will only resize twice in the worst case.+ let iter = iter.into_iter();+ let reserve = if self.is_empty() {+ iter.size_hint().0+ } else {+ iter.size_hint().0.div_ceil(2)+ };+ self.reserve(reserve);+ iter.for_each(move |wme| {+ self.insert(wme);+ });+ }+}++impl Index<usize> for WordMap {+ type Output = <CanonicalStorage as Index<usize>>::Output;++ fn index(&self, index: usize) -> &Self::Output {+ &self.canonical[index]+ }+}++impl IntoIterator for WordMap {+ type Item = WordMapEntry;+ type IntoIter = indexmap::map::IntoValues<CanonicalWordId, WordMapEntry>;++ fn into_iter(self) -> Self::IntoIter {+ self.canonical.into_values()+ }+}++impl FromIterator<WordMapEntry> for WordMap {+ fn from_iter<T: IntoIterator<Item = WordMapEntry>>(iter: T) -> Self {+ let mut out = Self::new();+ out.extend(iter);+ out+ }+}diff --git a/harper-core/src/spell/word_map/word_map_entry.rs b/harper-core/src/spell/word_map/word_map_entry.rsnew file mode 100644index 000000000..047ec195a--- /dev/null+++ b/harper-core/src/spell/word_map/word_map_entry.rs@@ -0,0 +1,35 @@+use crate::{CharString, DictWordMetadata, spell::word_id::WordIdPair};++#[derive(Debug, Clone, Eq, PartialEq, Hash)]+pub struct WordMapEntry {+ pub metadata: DictWordMetadata,+ pub canonical_spelling: CharString,+ pub word_ids: WordIdPair,+}++impl WordMapEntry {+ /// Construct a new word map entry with default metadata.+ pub fn new(canonical_spelling: impl Into<CharString>) -> Self {+ Self::from_chars(canonical_spelling.into())+ }++ /// Construct a new word map entry with default metadata.+ pub fn new_str(canonical_spelling: &str) -> Self {+ Self::from_chars(canonical_spelling.chars().collect())+ }++ fn from_chars(canonical_spelling: CharString) -> Self {+ let word_ids = WordIdPair::from_word_chars(&canonical_spelling);++ Self {+ metadata: Default::default(),+ canonical_spelling,+ word_ids,+ }+ }++ pub fn with_md(mut self, metadata: DictWordMetadata) -> Self {+ self.metadata = metadata;+ self+ }+}diff --git a/harper-core/tests/run_tests.rs b/harper-core/tests/run_tests.rsindex 9341420ca..e6ef8c9f6 100644--- a/harper-core/tests/run_tests.rs+++ b/harper-core/tests/run_tests.rs@@ -79,7 +79,7 @@ create_test!(issue_159.md, 1, Dialect::American);create_test!(issue_358.md, 0, Dialect::American);create_test!(issue_195.md, 0, Dialect::American);create_test!(issue_118.md, 0, Dialect::American);-create_test!(lots_of_latin.md, 1, Dialect::American);+create_test!(lots_of_latin.md, 0, Dialect::American);create_test!(pr_504.md, 1, Dialect::American);create_test!(pr_452.md, 2, Dialect::American);create_test!(hex_basic_clean.md, 0, Dialect::American);@@ -105,4 +105,4 @@ create_test!(issue_2151.md, 3, Dialect::British);create_test!(lukas_homework.md, 4, Dialect::American);// Org mode tests-create_org_test!(index.org, 49, Dialect::American);+create_org_test!(index.org, 48, Dialect::American);diff --git a/harper-core/tests/text/linters/Alice's Adventures in Wonderland.snap.yml b/harper-core/tests/text/linters/Alice's Adventures in Wonderland.snap.ymlindex bf2c14e85..681b36576 100644--- a/harper-core/tests/text/linters/Alice's Adventures in Wonderland.snap.yml+++ b/harper-core/tests/text/linters/Alice's Adventures in Wonderland.snap.yml@@ -625,26 +625,6 @@ Suggest:-Lint: Capitalization (127 priority)-Message: |- 320 | she began again: “Où est ma chatte?” which was the first sentence in her French- | ^~~ This word's canonical spelling is all-caps.-Suggest:- - Replace with: “EST”----Lint: Spelling (63 priority)-Message: |- 320 | she began again: “Où est ma chatte?” which was the first sentence in her French- | ^~~ Did you mean to spell `est` this way?-Suggest:- - Replace with: “east”- - Replace with: “eat”- - Replace with: “esp”---Lint: Spelling (63 priority)Message: |320 | she began again: “Où est ma chatte?” which was the first sentence in her French@@ -998,15 +978,6 @@ Message: |-Lint: Capitalization (31 priority)-Message: |- 607 | Alas! it was too late to wish that! She went on growing, and growing, and very- | ^~ This sentence does not start with a capital letter-Suggest:- - Replace with: “It”---Lint: Readability (127 priority)Message: |607 | Alas! it was too late to wish that! She went on growing, and growing, and very@@ -1454,15 +1425,6 @@ Suggest:-Lint: Capitalization (127 priority)-Message: |- 1059 | ## CHAPTER VI: Pig and Pepper- | ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Try to use title case in headings.-Suggest:- - Replace with: “## CHAPTER Vi: Pig and Pepper”---Lint: Readability (127 priority)Message: |1061 | For a minute or two she stood looking at the house, and wondering what to dodiff --git a/harper-core/tests/text/linters/Computer science.snap.yml b/harper-core/tests/text/linters/Computer science.snap.ymlindex c672cd12c..7d7e902e7 100644--- a/harper-core/tests/text/linters/Computer science.snap.yml+++ b/harper-core/tests/text/linters/Computer science.snap.yml@@ -46,8 +46,8 @@ Message: || ^~~~~~~~ Did you mean to spell `Reckoner` this way?Suggest:- Replace with: “Reckoned”- - Replace with: “Rickover”- Replace with: “Reasoner”+ - Replace with: “Rickover”@@ -598,7 +598,7 @@ Message: |Suggest:- Replace with: “Paras”- Replace with: “Parkas”- - Replace with: “Parana's”+ - Replace with: “Pan's”@@ -879,17 +879,6 @@ Suggest:-Lint: Spelling (63 priority)-Message: |- 262 | The famous P = NP? problem, one of the Millennium Prize Problems, is an open- | ^~ Did you mean to spell `NP` this way?-Suggest:- - Replace with: “Nap”- - Replace with: “Nip”- - Replace with: “No”---Lint: Capitalization (31 priority)Message: |262 | The famous P = NP? problem, one of the Millennium Prize Problems, is an open@@ -1051,8 +1040,8 @@ Message: || ^~~ Did you mean to spell `HCI` this way?Suggest:- Replace with: “Hi”- - Replace with: “H'm”- Replace with: “Chi”+ - Replace with: “Sci”@@ -1063,8 +1052,8 @@ Message: |349 | that focus on the relationship between emotions, social behavior and brainSuggest:- Replace with: “Hi”- - Replace with: “H'm”- Replace with: “Chi”+ - Replace with: “Sci”@@ -1120,7 +1109,7 @@ Message: |Suggest:- Replace with: “RI”- Replace with: “Ra”- - Replace with: “Ru”+ - Replace with: “Re”@@ -1321,9 +1310,9 @@ Message: |469 | goto (which means it is more elementary than structured programming).| ^~~~ Did you mean to spell `goto` this way?Suggest:+ - Replace with: “goth”- Replace with: “goo”- Replace with: “got”- - Replace with: “goths”diff --git a/harper-core/tests/text/linters/Part-of-speech tagging.snap.yml b/harper-core/tests/text/linters/Part-of-speech tagging.snap.ymlindex d3a9466db..e49b84812 100644--- a/harper-core/tests/text/linters/Part-of-speech tagging.snap.yml+++ b/harper-core/tests/text/linters/Part-of-speech tagging.snap.yml@@ -59,8 +59,8 @@ Message: || ^~~~~~~ Did you mean to spell `Brill's` this way?19 | first and most widely used English POS-taggers, employs rule-based algorithms.Suggest:- - Replace with: “Brillo's”- Replace with: “Bill's”+ - Replace with: “Brillo's”- Replace with: “Drill's”@@ -121,18 +121,6 @@ Suggest:-Lint: Spelling (63 priority)-Message: |- 49 | tags. For example, NN for singular common nouns, NNS for plural common nouns, NP- | ^~ Did you mean to spell `NP` this way?- 50 | for singular proper nouns (see the POS tags used in the Brown Corpus). Other-Suggest:- - Replace with: “Nap”- - Replace with: “Nip”- - Replace with: “No”---Lint: Spelling (63 priority)Message: |55 | 150 separate parts of speech for English. Work on stochastic methods for tagging@@ -605,7 +593,7 @@ Message: |Suggest:- Replace with: “Sim”- Replace with: “Sum”- - Replace with: “SCM”+ - Replace with: “SAM”diff --git a/harper-core/tests/text/linters/The Constitution of the United States.snap.yml b/harper-core/tests/text/linters/The Constitution of the United States.snap.ymlindex bb4086ea5..b8fa8d8d5 100644--- a/harper-core/tests/text/linters/The Constitution of the United States.snap.yml+++ b/harper-core/tests/text/linters/The Constitution of the United States.snap.yml@@ -1718,15 +1718,6 @@ Message: |-Lint: Capitalization (127 priority)-Message: |- 677 | ## Article. VI.- | ^~~~~~~~~~~~~~~ Try to use title case in headings.-Suggest:- - Replace with: “## Article. Vi.”---Lint: Readability (127 priority)Message: |683 | This Constitution, and the Laws of the United States which shall be made indiff --git a/harper-core/tests/text/linters/The Great Gatsby.snap.yml b/harper-core/tests/text/linters/The Great Gatsby.snap.ymlindex 6e36711de..8d6fc60cb 100644--- a/harper-core/tests/text/linters/The Great Gatsby.snap.yml+++ b/harper-core/tests/text/linters/The Great Gatsby.snap.yml@@ -882,7 +882,7 @@ Message: |Suggest:- Replace with: “Be”- Replace with: “Bi”- - Replace with: “BO”+ - Replace with: “BA”@@ -967,7 +967,7 @@ Message: |Suggest:- Replace with: “Do”- Replace with: “DA”- - Replace with: “Di”+ - Replace with: “DE”@@ -1133,7 +1133,7 @@ Message: |Suggest:- Replace with: “Be”- Replace with: “Bi”- - Replace with: “BO”+ - Replace with: “BA”@@ -2215,7 +2215,7 @@ Message: |Suggest:- Replace with: “RI”- Replace with: “Ra”- - Replace with: “Ru”+ - Replace with: “Re”@@ -2266,8 +2266,8 @@ Message: |1861 | the gravel drive that Mrs. Ulysses Swett’s automobile ran over his right hand.| ^~~~~~~ Did you mean to spell `Swett’s` this way?Suggest:- - Replace with: “Sweat's”- Replace with: “Sweet's”+ - Replace with: “Sweat's”- Replace with: “Seth's”@@ -2290,7 +2290,7 @@ Message: |Suggest:- Replace with: “So”- Replace with: “SA”- - Replace with: “Se”+ - Replace with: “SE”@@ -2301,7 +2301,7 @@ Message: |Suggest:- Replace with: “Be”- Replace with: “Bi”- - Replace with: “BO”+ - Replace with: “BA”@@ -2394,7 +2394,7 @@ Message: |Suggest:- Replace with: “So”- Replace with: “SA”- - Replace with: “Se”+ - Replace with: “SE”@@ -2440,8 +2440,8 @@ Message: || ^~ Did you mean to spell `G.` this way?Suggest:- Replace with: “Go”- - Replace with: “GI”- - Replace with: “GU”+ - Replace with: “GA”+ - Replace with: “GE”@@ -2517,7 +2517,7 @@ Message: |Suggest:- Replace with: “Lear's”- Replace with: “Leger's”- - Replace with: “Leer's”+ - Replace with: “Lego's”@@ -2529,7 +2529,7 @@ Message: |Suggest:- Replace with: “Be”- Replace with: “Bi”- - Replace with: “BO”+ - Replace with: “BA”@@ -2541,7 +2541,7 @@ Message: |Suggest:- Replace with: “Jon's”- Replace with: “Jonas”- - Replace with: “Jogs”+ - Replace with: “Jones”@@ -2692,7 +2692,7 @@ Message: |Suggest:- Replace with: “So”- Replace with: “SA”- - Replace with: “Se”+ - Replace with: “SE”@@ -2748,9 +2748,9 @@ Message: |1882 | and Henry L. Palmetto, who killed himself by jumping in front of a subway train| ^~ Did you mean to spell `L.` this way?Suggest:+ - Replace with: “LA”- Replace with: “Le”- Replace with: “Li”- - Replace with: “Lu”@@ -2924,8 +2924,8 @@ Message: |1982 | de Boulogne.Suggest:- Replace with: “Boss”+ - Replace with: “Boas”- Replace with: “Boris”- - Replace with: “Bios”@@ -2962,16 +2962,6 @@ Suggest:-Lint: Capitalization (127 priority)-Message: |- 2006 | To my astonishment, the thing had an authentic look. “Orderi di Danilo,” ran the- | ^~ This word's canonical spelling is all-caps.- 2007 | circular legend, “Montenegro, Nicolas Rex.”-Suggest:- - Replace with: “DI”---Lint: Spelling (63 priority)Message: |2006 | To my astonishment, the thing had an authentic look. “Orderi di Danilo,” ran the@@ -4120,15 +4110,6 @@ Suggest:-Lint: Capitalization (127 priority)-Message: |- 3003 | ## CHAPTER VI- | ^~~~~~~~~~~~~ Try to use title case in headings.-Suggest:- - Replace with: “## CHAPTER Vi”---Lint: Readability (127 priority)Message: |3020 | short of being news. Contemporary legends such as the “underground pipe-line to@@ -5165,7 +5146,7 @@ Message: |Suggest:- Replace with: “Biko's”- Replace with: “Bilbo's”- - Replace with: “Biro's”+ - Replace with: “Bill's”@@ -5176,7 +5157,7 @@ Message: |Suggest:- Replace with: “Biko's”- Replace with: “Bilbo's”- - Replace with: “Biro's”+ - Replace with: “Bill's”@@ -5187,7 +5168,7 @@ Message: |Suggest:- Replace with: “Biko's”- Replace with: “Bilbo's”- - Replace with: “Biro's”+ - Replace with: “Bill's”@@ -5199,7 +5180,7 @@ Message: |Suggest:- Replace with: “Biko's”- Replace with: “Bilbo's”- - Replace with: “Biro's”+ - Replace with: “Bill's”@@ -5217,7 +5198,7 @@ Message: |Suggest:- Replace with: “Biko's”- Replace with: “Bilbo's”- - Replace with: “Biro's”+ - Replace with: “Bill's”@@ -5270,7 +5251,7 @@ Message: |Suggest:- Replace with: “Biko's”- Replace with: “Bilbo's”- - Replace with: “Biro's”+ - Replace with: “Bill's”@@ -5288,7 +5269,7 @@ Message: |Suggest:- Replace with: “Biko's”- Replace with: “Bilbo's”- - Replace with: “Biro's”+ - Replace with: “Bill's”@@ -5311,7 +5292,7 @@ Message: |Suggest:- Replace with: “Biko's”- Replace with: “Bilbo's”- - Replace with: “Biro's”+ - Replace with: “Bill's”@@ -5322,7 +5303,7 @@ Message: |Suggest:- Replace with: “Biko's”- Replace with: “Bilbo's”- - Replace with: “Biro's”+ - Replace with: “Bill's”@@ -5758,8 +5739,8 @@ Message: || ^ Did you mean to spell `r` this way?Suggest:- Replace with: “re”+ - Replace with: “I”- Replace with: “a”- - Replace with: “i”@@ -5769,8 +5750,8 @@ Message: || ^ Did you mean to spell `r` this way?Suggest:- Replace with: “re”+ - Replace with: “I”- Replace with: “a”- - Replace with: “i”@@ -5800,8 +5781,8 @@ Message: || ^ Did you mean to spell `r` this way?Suggest:- Replace with: “re”+ - Replace with: “I”- Replace with: “a”- - Replace with: “i”@@ -6638,9 +6619,9 @@ Message: |5172 | Roosevelt and then to Gad’s Hill, where he bought a sandwich that he didn’t eat,| ^~~~~ Did you mean to spell `Gad’s` this way?Suggest:- - Replace with: “Gab's”- - Replace with: “Gag's”- - Replace with: “Gal's”+ - Replace with: “Gap's”+ - Replace with: “Gay's”+ - Replace with: “God's”@@ -6650,9 +6631,9 @@ Message: |5174 | reach Gad’s Hill until noon. Thus far there was no difficulty in accounting for| ^~~~~ Did you mean to spell `Gad’s` this way?Suggest:- - Replace with: “Gab's”- - Replace with: “Gag's”- - Replace with: “Gal's”+ - Replace with: “Gap's”+ - Replace with: “Gay's”+ - Replace with: “God's”@@ -6917,9 +6898,9 @@ Message: |5341 | “Young Parke’s in trouble,” he said rapidly. “They picked him up when he handed| ^~~~~~~ Did you mean to spell `Parke’s` this way?Suggest:+ - Replace with: “Park's”- Replace with: “Parker's”- Replace with: “Parks's”- - Replace with: “Park's”@@ -6959,7 +6940,7 @@ Message: |Suggest:- Replace with: “Co”- Replace with: “Cu”- - Replace with: “CI”+ - Replace with: “CA”@@ -7107,7 +7088,7 @@ Message: |Suggest:- Replace with: “Be”- Replace with: “Bi”- - Replace with: “BO”+ - Replace with: “BA”@@ -7311,8 +7292,8 @@ Message: || ^~~~~~~~ Did you mean to spell `Hopalong` this way?Suggest:- Replace with: “Hopping”+ - Replace with: “Halon”- Replace with: “Haling”- - Replace with: “Haloing”diff --git a/harper-core/tests/text/tagged/Alice's Adventures in Wonderland.md b/harper-core/tests/text/tagged/Alice's Adventures in Wonderland.mdindex 4ca5e1ebc..4b7dcf6b0 100644--- a/harper-core/tests/text/tagged/Alice's Adventures in Wonderland.md+++ b/harper-core/tests/text/tagged/Alice's Adventures in Wonderland.md@@ -1161,7 +1161,7 @@> RABBIT , ” engraved upon it . She went in without knocking , and hurried upstairs ,# NSg/VB+ . . VP/J P NSg/ISg+ . ISg+ VPt NPr/J/R/P C/P Nᴹ/Vg/J . VB/C VP/J NSg/J .> in great fear lest she should meet the real Mary Ann , and be turned out of the-# NPr/J/R/P NSg/J+ N🅪Sg/VB+ JS ISg+ VXB NSg/VB/J D+ NSg/J+ NPr+ NPr/J+ . VB/C NSg/VLXB VP/J NSg/VB/J/R/P P D++# NPr/J/R/P NSg/J+ N🅪Sg/VB+ W? ISg+ VXB NSg/VB/J D+ NSg/J+ NPr+ NPr/J+ . VB/C NSg/VLXB VP/J NSg/VB/J/R/P P D+> house before she had found the fan and gloves .# NPr/VB+ C/P ISg+ VP NSg/VP D NSg/VB VB/C NPl/V3+ .>@@ -5699,7 +5699,7 @@> must have meant some mischief , or else you’d have signed your name like an# NSg/VXB NSg/VXB VP I/J/R/Dq+ NSg/VB+ . NPr/C NSg/J/C K NSg/VXB VP/J D$+ NSg/VB+ NSg/VB/J/C/P D/P> honest man . ”-# VB/JS NPr/VB+ . .+# VB/J NPr/VB+ . .>#> There was a general clapping of hands at this : it was the first really cleverdiff --git a/harper-core/tests/text/tagged/The Great Gatsby.md b/harper-core/tests/text/tagged/The Great Gatsby.mdindex d26208e63..cd32c8845 100644--- a/harper-core/tests/text/tagged/The Great Gatsby.md+++ b/harper-core/tests/text/tagged/The Great Gatsby.md@@ -811,7 +811,7 @@>#> “ Well , he wasn’t always a butler ; he used to be the silver polisher for some-# . NSg/VB/J/R . NPr/ISg+ VPt R D/P NPr/VB . NPr/ISg+ VP/J P NSg/VLXB D Nᴹ/VB/J+ NSg/JC R/C/P I/J/R/Dq+# . NSg/VB/J/R . NPr/ISg+ VPt R D/P NPr/VB . NPr/ISg+ VP/J P NSg/VLXB D Nᴹ/VB/J+ NSg R/C/P I/J/R/Dq> people in New York that had a silver service for two hundred people . He had to# NPl/VB+ NPr/J/R/P NSg/J NPr+ I/C/Ddem+ VP D/P Nᴹ/VB/J+ NSg/VB+ R/C/P NSg NSg NPl/VB+ . NPr/ISg+ VP P> polish it from morning till night , until finally it began to affect his nose — — — ”@@ -3653,7 +3653,7 @@> Every one suspects himself of at least one of the cardinal virtues , and this is# Dq NSg/I/J NPl/V3 ISg+ P NSg/P NSg/J/Dq NSg/I/J P D+ NSg/J+ NPl+ . VB/C I/Ddem+ VL3> mine : I am one of the few honest people that I have ever known .-# NSg/I/VB+ . ISg/#r+ NPr/VLB/J NSg/I/J P D+ NSg/I/Dq+ VB/JS+ NPl/VB+ I/C/Ddem+ ISg/#r+ NSg/VXB J/R VPp/J .+# NSg/I/VB+ . ISg/#r+ NPr/VLB/J NSg/I/J P D+ NSg/I/Dq+ VB/J+ NPl/VB+ I/C/Ddem+ ISg/#r+ NSg/VXB J/R VPp/J .>#> CHAPTER IV@@ -11463,7 +11463,7 @@> met another bad driver , didn’t I ? I mean it was careless of me to make such a# VP I/D+ NSg/VB/J+ NSg+ . VXPt ISg/#r+ . ISg/#r+ NSg/VB/J NSg/ISg+ VLPt J P NPr/ISg+ P NSg/VB NSg/I D/P> wrong guess . I thought you were rather an honest , straightforward person . I-# NSg/VB/J/R NSg/VB+ . ISg/#r+ N🅪Sg/VP ISgPl+ VLPt NPr/R D/P VB/JS . J+ NSg/VB+ . ISg/#r++# NSg/VB/J/R NSg/VB+ . ISg/#r+ N🅪Sg/VP ISgPl+ VLPt NPr/R D/P VB/J . J+ NSg/VB+ . ISg/#r+> thought it was your secret pride . ”# N🅪Sg/VP NSg/ISg+ VLPt D$+ NSg/VB/J+ Nᴹ/VB+ . .>diff --git a/harper-desktop/src-tauri/Cargo.lock b/harper-desktop/src-tauri/Cargo.lockindex e04041427..936d4e687 100644--- a/harper-desktop/src-tauri/Cargo.lock+++ b/harper-desktop/src-tauri/Cargo.lock@@ -3785,6 +3785,7 @@ dependencies = ["harper-brill","harper-thesaurus","hashbrown 0.16.1",+ "indexmap 2.14.2","is-macro","itertools 0.15.0","levenshtein_automata",