Skip to content

Offline, privacy-first grammar checker. Fast, open-source, Rust-powered

feat(core)!: more explicit handling of case-sensitivity in dictionaries (#2630)

Steve Kurch760378b7ab40
diff --git a/Cargo.lock b/Cargo.lock
index c2ac994e2..e06d66827 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -2724,6 +2724,7 @@ dependencies = [
"harper-brill",
"harper-thesaurus",
"hashbrown 0.16.1",
+ "indexmap",
"is-macro",
"itertools 0.15.0",
"levenshtein_automata",
diff --git a/harper-cli/src/main.rs b/harper-cli/src/main.rs
index 519b1f09d..2d314b5cc 100644
--- a/harper-cli/src/main.rs
+++ b/harper-cli/src/main.rs
@@ -21,7 +21,7 @@ use harper_core::{
CharStringExt, Dialect, DictWordMetadata, OrthFlags, Span, TokenKind, TokenStringExt,
linting::LintGroup,
parsers::{IsolateEnglish, MarkdownOptions},
- spell::{Dictionary, FstDictionary, MutableDictionary, WordId},
+ spell::{CanonicalWordId, Dictionary, FstDictionary, MutableDictionary},
weir::WeirLinter,
};
#[cfg(feature = "training")]
@@ -1065,11 +1065,15 @@ fn line_to_parts(line: &str) -> (String, String) {
fn print_word_derivations(word: &str, annot: &str, dictionary: &impl Dictionary) {
println!("{word}/{annot}");
- let id = WordId::from_word_str(word);
+ let id = CanonicalWordId::from_word_str(word);
- let children = dictionary
- .words_iter()
- .filter(|e| dictionary.get_word_metadata(e).unwrap().derived_from == Some(id));
+ let children = dictionary.words_iter().filter(|word| {
+ dictionary
+ .get_word_metadata_exact(word)
+ .unwrap()
+ .derived_from
+ .contains(id)
+ });
println!(" - {word}");
diff --git a/harper-core/Cargo.toml b/harper-core/Cargo.toml
index 33fff3434..bd3b0d8e0 100644
--- a/harper-core/Cargo.toml
+++ b/harper-core/Cargo.toml
@@ -34,6 +34,7 @@ harper-brill = { path = "../harper-brill", version = "2.0.0" }
harper-thesaurus = { path = "../harper-thesaurus", version = "2.0.0", optional = true }
bitflags = { version = "2.11.0", features = ["serde"] }
trie-rs = "0.4.2"
+indexmap = "2.13.1"
zip = { version = "8.6.0", default-features = false, features = ["deflate"] }
regex = "1.12.3"
boxcar = { version = "0.2.14", optional = true }
diff --git a/harper-core/dictionary.dict b/harper-core/dictionary.dict
index 94247fcb6..53f4670b1 100644
--- a/harper-core/dictionary.dict
+++ b/harper-core/dictionary.dict
@@ -8311,6 +8311,7 @@ PowerPoint/ONgV
Powers/NOg
Powhatan/NOg
Poznan/Og
+Pr/ # Praseodymium
Prada/g
Prado/Og
Praetorian/Ng
diff --git a/harper-core/src/dict_word_metadata.rs b/harper-core/src/dict_word_metadata.rs
index acbebdf80..98505caf6 100644
--- a/harper-core/src/dict_word_metadata.rs
+++ b/harper-core/src/dict_word_metadata.rs
@@ -10,9 +10,11 @@ use strum_macros::{Display, EnumCount, EnumIter, EnumString, VariantArray};
use std::convert::TryFrom;
use crate::dict_word_metadata_orthography::OrthFlags;
-use crate::spell::WordId;
use crate::{Document, TokenKind, TokenStringExt};
+pub mod derived_from;
+use derived_from::DerivedFrom;
+
/// This represents a "lexeme" or "headword" which is case-folded but affix-expanded.
/// So not only lemmata but also inflected forms are stored here, with "horn" and "horns" each
/// having their own lexeme, but "Ivy" and "ivy" sharing the same lexeme.
@@ -46,8 +48,8 @@ pub struct DictWordMetadata {
/// Whether the word is considered especially common.
#[serde(default = "default_false")]
pub common: bool,
- #[serde(default = "default_none")]
- pub derived_from: Option<WordId>,
+ #[serde(default = "DerivedFrom::default")]
+ pub derived_from: DerivedFrom,
/// Generated by a chunker. Declares whether the word is a member of a nominal phrase. Using
/// this should be preferred over the similarly named `Pattern`.
///
@@ -62,11 +64,6 @@ fn default_false() -> bool {
false
}
-/// Needed for `serde`
-fn default_none<T>() -> Option<T> {
- None
-}
-
/// Needed for `serde`
fn default_default<T: Default>() -> T {
T::default()
@@ -863,7 +860,7 @@ impl DictWordMetadata {
self.swear = self.swear.or(other.swear);
self.abbreviation = self.abbreviation.or(other.abbreviation);
self.common |= other.common;
- self.derived_from = self.derived_from.or(other.derived_from);
+ self.derived_from.extend(other.derived_from.iter());
self.pos_tag = self.pos_tag.or(other.pos_tag);
self.np_member = self.np_member.or(other.np_member);
diff --git a/harper-core/src/dict_word_metadata/derived_from.rs b/harper-core/src/dict_word_metadata/derived_from.rs
new file mode 100644
index 000000000..044de1b04
--- /dev/null
+++ b/harper-core/src/dict_word_metadata/derived_from.rs
@@ -0,0 +1,73 @@
+use std::{
+ collections::{BTreeSet, btree_set},
+ iter::Extend,
+};
+
+use serde::{Deserialize, Serialize};
+
+use crate::spell::CanonicalWordId;
+
+/// A container for storing word IDs that a word is considered to be derived from.
+#[derive(Debug, Default, Clone, PartialEq, Eq, Serialize, Deserialize, PartialOrd, Hash)]
+pub struct DerivedFrom {
+ inner: BTreeSet<CanonicalWordId>,
+}
+
+impl DerivedFrom {
+ /// Insert another word ID, if it's not already contained.
+ ///
+ /// If it is already contained, it's quietly ignored.
+ pub fn insert(&mut self, id: CanonicalWordId) {
+ self.inner.insert(id);
+ }
+
+ /// Is the list empty? In other words, Does this word have no known words it's derived from?
+ pub fn is_empty(&self) -> bool {
+ self.inner.is_empty()
+ }
+
+ /// Is the list empty? In other words, Does this word have no known words it's derived from?
+ /// This is identical to `is_empty`. It's only here for backward-compatibility.
+ pub fn is_none(&self) -> bool {
+ self.is_empty()
+ }
+
+ /// Is this word derived from the word represented by `id`?
+ pub fn contains(&self, id: CanonicalWordId) -> bool {
+ self.inner.contains(&id)
+ }
+
+ /// Create a new `DerivedFrom` containing a single initial word ID.
+ pub fn from_canonical_word_id(word_id: CanonicalWordId) -> Self {
+ Self {
+ inner: BTreeSet::from([word_id]),
+ }
+ }
+
+ /// Get an iterator of the contained [`CanonicalWordId`].
+ pub fn iter(&self) -> btree_set::Iter<'_, CanonicalWordId> {
+ self.inner.iter()
+ }
+}
+
+impl Extend<CanonicalWordId> for DerivedFrom {
+ fn extend<T: IntoIterator<Item = CanonicalWordId>>(&mut self, iter: T) {
+ self.inner.extend(iter);
+ }
+}
+
+impl<'a> Extend<&'a CanonicalWordId> for DerivedFrom {
+ fn extend<T: IntoIterator<Item = &'a CanonicalWordId>>(&mut self, iter: T) {
+ self.inner.extend(iter);
+ }
+}
+
+impl IntoIterator for DerivedFrom {
+ type Item = CanonicalWordId;
+
+ type IntoIter = btree_set::IntoIter<Self::Item>;
+
+ fn into_iter(self) -> Self::IntoIter {
+ self.inner.into_iter()
+ }
+}
diff --git a/harper-core/src/dict_word_metadata_orthography.rs b/harper-core/src/dict_word_metadata_orthography.rs
index 3b0254c0b..0d6411bbf 100644
--- a/harper-core/src/dict_word_metadata_orthography.rs
+++ b/harper-core/src/dict_word_metadata_orthography.rs
@@ -54,6 +54,18 @@ impl Default for OrthFlags {
}
impl OrthFlags {
+ /// An `OrthFlags` with all casing-related flags set.
+ pub const CASE_FLAGS: OrthFlags = OrthFlags::LOWERCASE
+ .union(OrthFlags::TITLECASE)
+ .union(OrthFlags::ALLCAPS)
+ .union(OrthFlags::LOWER_CAMEL)
+ .union(OrthFlags::UPPER_CAMEL);
+
+ /// Mask out all flags not related to casing.
+ pub const fn case_flags(self) -> Self {
+ self.intersection(Self::CASE_FLAGS)
+ }
+
/// Construct orthography flags for a given sequence of letters.
pub fn from_letters(letters: &[char]) -> Self {
let mut ortho_flags = Self::default();
diff --git a/harper-core/src/linting/orthographic_consistency.rs b/harper-core/src/linting/orthographic_consistency.rs
index 48ea41cd7..bf85de99d 100644
--- a/harper-core/src/linting/orthographic_consistency.rs
+++ b/harper-core/src/linting/orthographic_consistency.rs
@@ -66,12 +66,16 @@ impl ExprLinter for OrthographicConsistency {
let chars = word.get_ch(source);
+ if self.dict.contains_exact_word(chars) {
+ // Exit if the dictionary contains the exact word.
+ return None;
+ }
+
let cur_flags = OrthFlags::from_letters(chars);
if metadata.is_allcaps()
- && !metadata.is_lowercase()
- && !metadata.is_upper_camel()
&& !cur_flags.contains(OrthFlags::ALLCAPS)
+ && metadata.orth_info.case_flags().bits().count_ones() == 1
{
return Some(Lint {
span: word.span,
@@ -101,7 +105,7 @@ impl ExprLinter for OrthographicConsistency {
&& !canonical_flags.contains(OrthFlags::TITLECASE)
})
.count()
- == 1
+ != 0
&& let Some(canonical) = self.dict.get_correct_capitalization_of(chars)
&& alphabetic_differs(canonical, chars)
{
@@ -148,7 +152,10 @@ fn alphabetic_differs(a: &[char], b: &[char]) -> bool {
#[cfg(test)]
mod tests {
- use crate::linting::tests::{assert_no_lints, assert_suggestion_result};
+ use crate::linting::tests::{
+ assert_good_and_bad_suggestions, assert_lint_count, assert_no_lints,
+ assert_suggestion_result,
+ };
use super::OrthographicConsistency;
@@ -170,15 +177,6 @@ mod tests {
);
}
- #[test]
- fn lego_should_be_all_caps() {
- assert_suggestion_result(
- "Lego bricks encourage creativity.",
- OrthographicConsistency::default(),
- "LEGO bricks encourage creativity.",
- );
- }
-
#[test]
fn nato_should_be_all_caps() {
assert_suggestion_result(
@@ -453,6 +451,33 @@ mod tests {
);
}
+ #[test]
+ fn no_improper_suggestion_for_macos() {
+ assert_good_and_bad_suggestions(
+ "MacOS",
+ OrthographicConsistency::default(),
+ &["macOS"],
+ &["MacOS"],
+ );
+ }
+
+ #[test]
+ fn accept_case_variants() {
+ // At the time of writing this test, "Pr" (despite being a word in the curated dictionary)
+ // would be linted for the supposed reason of the canonical spelling being "PR".
+ // Since both words are in the curated dictionary, neither should be linted.
+ assert_no_lints("Pr PR", OrthographicConsistency::default());
+ assert_no_lints("Apr APR", OrthographicConsistency::default()); // https://github.com/Automattic/harper/issues/3080
+ assert_no_lints("install.md", OrthographicConsistency::default()); // https://github.com/Automattic/harper/issues/3073
+ }
+
+ #[test]
+ #[ignore = "Words with multiple known orthographic variants are not currently supported"]
+ fn dont_accept_undefined_case_variants() {
+ // "pr" isn't defined in the dictionary, so it should be linted.
+ assert_lint_count("pr", OrthographicConsistency::default(), 1);
+ }
+
#[test]
fn lets_confusion_description() {
assert_no_lints(
diff --git a/harper-core/src/linting/sentence_capitalization.rs b/harper-core/src/linting/sentence_capitalization.rs
index b49d1ef68..a514118ac 100644
--- a/harper-core/src/linting/sentence_capitalization.rs
+++ b/harper-core/src/linting/sentence_capitalization.rs
@@ -37,7 +37,7 @@ impl<T: Dictionary> Linter for SentenceCapitalization<T> {
}
}
- for sentence in paragraph.iter_sentences() {
+ 'sentence: for sentence in paragraph.iter_sentences() {
if !is_full_sentence(sentence) {
continue;
}
@@ -53,12 +53,12 @@ impl<T: Dictionary> Linter for SentenceCapitalization<T> {
&& first_char.is_alphabetic()
&& !first_char.is_uppercase()
{
- if let Some(canonical_spelling) =
- self.dictionary.get_correct_capitalization_of(word_chars)
+ for canonical_spelling in
+ self.dictionary.get_correct_capitalizations_of(word_chars)
{
// Skip if it's a proper noun or contains uppercase letters before a separator
if first_word.kind.is_proper_noun() {
- continue;
+ continue 'sentence;
}
// Check for uppercase letters in the rest of the word before any separators
@@ -68,7 +68,7 @@ impl<T: Dictionary> Linter for SentenceCapitalization<T> {
.take_while(|&c| !c.is_whitespace() && *c != '-' && *c != '\'')
.any(|&c| c.is_uppercase())
{
- continue;
+ continue 'sentence;
}
}
diff --git a/harper-core/src/linting/spell_check.rs b/harper-core/src/linting/spell_check.rs
index 792035657..3d9da5b97 100644
--- a/harper-core/src/linting/spell_check.rs
+++ b/harper-core/src/linting/spell_check.rs
@@ -1,5 +1,6 @@
use std::num::NonZero;
+use itertools::Itertools;
use lru::LruCache;
use smallvec::ToSmallVec;
@@ -53,6 +54,7 @@ impl<T: Dictionary> SpellCheck<T> {
.dialects
.is_dialect_enabled(self.dialect)
})
+ .unique_by(|v| v.to_lower())
.map(|v| v.to_smallvec())
.take(Self::MAX_SUGGESTIONS)
.collect();
@@ -100,7 +102,12 @@ impl<T: Dictionary> Linter for SpellCheck<T> {
if let Some(metadata) = word.kind.as_word().unwrap()
&& metadata.dialects.is_dialect_enabled(self.dialect)
&& (self.dictionary.contains_exact_word(word_chars)
- || self.dictionary.contains_exact_word(&word_chars.to_lower()))
+ || self
+ .dictionary
+ .contains_exact_word(word_chars.normalized().as_ref())
+ || self
+ .dictionary
+ .contains_exact_word(word_chars.normalized().to_lower().as_ref()))
{
continue;
};
@@ -169,7 +176,7 @@ mod tests {
use crate::dict_word_metadata::DialectFlags;
use crate::linting::Linter;
use crate::linting::pooled_linter::for_tests::create_test_pool;
- use crate::linting::tests::{assert_good_and_bad_suggestions, assert_no_lints};
+ use crate::linting::tests::assert_no_lints;
use crate::spell::{Dictionary, FstDictionary, MergedDictionary, MutableDictionary};
use crate::{
Dialect,
@@ -527,18 +534,10 @@ mod tests {
}
#[test]
- fn dont_flag_pr() {
+ fn dont_flag_certain_entries_with_multiple_case_variants_in_dict() {
assert_no_lints("PR", test_linter());
- }
-
- #[test]
- fn no_improper_suggestion_for_macos() {
- assert_good_and_bad_suggestions("MacOS", test_linter(), &["macOS"], &["MacOS"]);
- }
-
- #[test]
- fn allows_parenthetical_plural_s() {
- assert_no_lints("Please ask each person(s) to sign.", test_linter());
+ assert_no_lints("MB", test_linter());
+ assert_no_lints("OS", test_linter()); // Issue #2585
}
#[test]
diff --git a/harper-core/src/linting/split_words.rs b/harper-core/src/linting/split_words.rs
index 4ee9e0b81..4a61579f8 100644
--- a/harper-core/src/linting/split_words.rs
+++ b/harper-core/src/linting/split_words.rs
@@ -103,21 +103,21 @@ impl ExprLinter for SplitWords {
let remainder = &chars[split_pos..];
// Both parts must be valid common words
- let Some(cand_meta) = self.dict.get_word_metadata(candidate) else {
+ let Some(cand_meta) = self.dict.get_word_metadata_exact(candidate) else {
continue;
};
if !cand_meta.common {
continue;
}
- let Some(rem_meta) = self.dict.get_word_metadata(remainder) else {
+ let Some(rem_meta) = self.dict.get_word_metadata_exact(remainder) else {
continue;
};
if !rem_meta.common {
continue;
}
- if is_anchor_split(&cand_meta) || is_anchor_split(&rem_meta) {
+ if is_anchor_split(cand_meta) || is_anchor_split(rem_meta) {
has_anchor_split = true;
}
diff --git a/harper-core/src/patterns/derived_from.rs b/harper-core/src/patterns/derived_from.rs
index bb1d0ac93..17878fcc8 100644
--- a/harper-core/src/patterns/derived_from.rs
+++ b/harper-core/src/patterns/derived_from.rs
@@ -1,4 +1,4 @@
-use crate::spell::WordId;
+use crate::spell::{CanonicalWordId, CaseFoldedWordId};
use super::Pattern;
@@ -7,19 +7,19 @@ use super::Pattern;
///
/// For example, this will match "call" as well as "recall", "calling", etc.
pub struct DerivedFrom {
- word_id: WordId,
+ word_id: CanonicalWordId,
}
impl DerivedFrom {
pub fn new_from_str(word: &str) -> DerivedFrom {
- Self::new(WordId::from_word_str(word))
+ Self::new(CanonicalWordId::from_word_str(word))
}
pub fn new_from_chars(word: &[char]) -> DerivedFrom {
- Self::new(WordId::from_word_chars(word))
+ Self::new(CanonicalWordId::from_word_chars(word))
}
- pub fn new(word_id: WordId) -> Self {
+ pub fn new(word_id: CanonicalWordId) -> Self {
Self { word_id }
}
}
@@ -27,19 +27,21 @@ impl DerivedFrom {
impl Pattern for DerivedFrom {
fn matches(&self, tokens: &[crate::Token], source: &[char]) -> Option<usize> {
let tok = tokens.first()?;
- let metadata = tok.kind.as_word()?.as_ref()?;
-
- if metadata.derived_from == Some(self.word_id) {
- return Some(1);
- }
-
let chars = tok.get_ch(source);
- let word_id = WordId::from_word_chars(chars);
- if word_id == self.word_id {
- return Some(1);
+ let is_exact_match = CanonicalWordId::from_word_chars(chars) == self.word_id
+ || CaseFoldedWordId::from_word_chars(chars).0 == self.word_id.as_case_folded();
+
+ if is_exact_match
+ || tok
+ .kind
+ .as_word()?
+ .as_ref()
+ .is_some_and(|meta| meta.derived_from.contains(self.word_id))
+ {
+ Some(1)
+ } else {
+ None
}
-
- None
}
}
diff --git a/harper-core/src/spell/dictionary.rs b/harper-core/src/spell/dictionary.rs
index f06a8f92b..d82c74b10 100644
--- a/harper-core/src/spell/dictionary.rs
+++ b/harper-core/src/spell/dictionary.rs
@@ -1,10 +1,13 @@
use blanket::blanket;
use std::borrow::Cow;
+use super::CanonicalWordId;
use super::FuzzyMatchResult;
-use super::WordId;
use crate::DictWordMetadata;
+pub static CURATED_DICT_STR: &str = include_str!("../../dictionary.dict");
+pub static ANNOTATIONS_STR: &str = include_str!("../../annotations.json");
+
/// An in-memory database that contains everything necessary to parse and analyze English text.
///
/// See also: [`super::FstDictionary`] and [`super::MutableDictionary`].
@@ -32,7 +35,13 @@ pub trait Dictionary: Send + Sync {
max_distance: u8,
max_results: usize,
) -> Vec<FuzzyMatchResult<'_>>;
+ /// Gets the word's canonical capitalization, if exactly one exists.
+ /// If there are multiple possibilities, `None` is returned.
fn get_correct_capitalization_of(&self, word: &[char]) -> Option<&'_ [char]>;
+ fn get_correct_capitalizations_of(
+ &self,
+ word: &[char],
+ ) -> Box<dyn Iterator<Item = &[char]> + Send + '_>;
/// Get the associated [`DictWordMetadata`] for any capitalization of a given word.
fn get_word_metadata(&self, word: &[char]) -> Option<Cow<'_, DictWordMetadata>>;
/// Get the associated [`DictWordMetadata`] for any capitalization of a given word.
@@ -40,6 +49,8 @@ pub trait Dictionary: Send + Sync {
/// empty.
fn get_word_metadata_str(&self, word: &str) -> Option<Cow<'_, DictWordMetadata>>;
+ fn get_word_metadata_exact(&self, word: &[char]) -> Option<&DictWordMetadata>;
+
/// Iterate over the words in the dictionary.
fn words_iter(&self) -> Box<dyn Iterator<Item = &'_ [char]> + Send + '_>;
@@ -47,7 +58,7 @@ pub trait Dictionary: Send + Sync {
fn word_count(&self) -> usize;
/// Returns the correct capitalization of the word with the given ID.
- fn get_word_from_id(&self, id: &WordId) -> Option<&[char]>;
+ fn get_word_from_id(&self, id: &CanonicalWordId) -> Option<&[char]>;
/// Look for words with a specific prefix
fn find_words_with_prefix(&self, prefix: &[char]) -> Vec<Cow<'_, [char]>>;
diff --git a/harper-core/src/spell/fst_dictionary.rs b/harper-core/src/spell/fst_dictionary.rs
index e4d9cffa6..f28df671c 100644
--- a/harper-core/src/spell/fst_dictionary.rs
+++ b/harper-core/src/spell/fst_dictionary.rs
@@ -8,7 +8,7 @@ use fst::{IntoStreamer, Map as FstMap, Streamer, map::StreamWithState};
use hashbrown::HashMap;
use levenshtein_automata::{DFA, LevenshteinAutomatonBuilder};
-use super::{Dictionary, FuzzyMatchResult, MutableDictionary, WordId};
+use super::{CanonicalWordId, Dictionary, FuzzyMatchResult, MutableDictionary};
use crate::{CharString, CharStringExt, DictWordMetadata};
/// An immutable dictionary allowing for very fast spellchecking.
@@ -62,7 +62,7 @@ impl FstDictionary {
for (word_chars, _) in words.iter() {
let word = word_chars.iter().collect::<String>();
builder
- .insert(word, WordId::from_word_chars(word_chars).into())
+ .insert(word, CanonicalWordId::from_word_chars(word_chars).into())
.expect("Insertion not in lexicographical order!");
}
@@ -139,6 +139,17 @@ impl Dictionary for FstDictionary {
self.mutable_dict.get_word_metadata_str(word)
}
+ fn get_word_metadata_exact(&self, word: &[char]) -> Option<&DictWordMetadata> {
+ self.mutable_dict.get_word_metadata_exact(word)
+ }
+
+ fn get_correct_capitalizations_of(
+ &self,
+ word: &[char],
+ ) -> Box<dyn Iterator<Item = &[char]> + Send + '_> {
+ self.mutable_dict.get_correct_capitalizations_of(word)
+ }
+
fn fuzzy_match(
&'_ self,
word: &[char],
@@ -177,11 +188,11 @@ impl Dictionary for FstDictionary {
let mut merged = Vec::with_capacity(best_distances.len());
for (word_id, edit_distance) in best_distances {
let word = self.mutable_dict.get_word_from_id(&word_id.into()).unwrap();
- let metadata = self.mutable_dict.get_word_metadata(word).unwrap();
+ let metadata = self.mutable_dict.get_word_metadata_exact(word).unwrap();
merged.push(FuzzyMatchResult {
word,
edit_distance,
- metadata,
+ metadata: Cow::Borrowed(metadata),
});
}
@@ -230,7 +241,7 @@ impl Dictionary for FstDictionary {
self.mutable_dict.get_correct_capitalization_of(word)
}
- fn get_word_from_id(&self, id: &WordId) -> Option<&[char]> {
+ fn get_word_from_id(&self, id: &CanonicalWordId) -> Option<&[char]> {
self.mutable_dict.get_word_from_id(id)
}
@@ -249,7 +260,7 @@ mod tests {
use crate::CharStringExt;
use crate::DictWordMetadata;
- use crate::spell::{Dictionary, MutableDictionary, WordId};
+ use crate::spell::{CanonicalWordId, Dictionary, MutableDictionary};
use super::FstDictionary;
@@ -387,12 +398,11 @@ mod tests {
fn plural_llamas_derived_from_llama() {
let dict = FstDictionary::curated();
- assert_eq!(
+ assert!(
dict.get_word_metadata_str("llamas")
.unwrap()
.derived_from
- .unwrap(),
- WordId::from_word_str("llama")
+ .contains(CanonicalWordId::from_word_str("llama"))
)
}
@@ -400,12 +410,11 @@ mod tests {
fn plural_cats_derived_from_cat() {
let dict = FstDictionary::curated();
- assert_eq!(
+ assert!(
dict.get_word_metadata_str("cats")
.unwrap()
.derived_from
- .unwrap(),
- WordId::from_word_str("cat")
+ .contains(CanonicalWordId::from_word_str("cat"))
);
}
@@ -413,12 +422,11 @@ mod tests {
fn unhappy_derived_from_happy() {
let dict = FstDictionary::curated();
- assert_eq!(
+ assert!(
dict.get_word_metadata_str("unhappy")
.unwrap()
.derived_from
- .unwrap(),
- WordId::from_word_str("happy")
+ .contains(CanonicalWordId::from_word_str("happy"))
);
}
@@ -426,12 +434,11 @@ mod tests {
fn quickly_derived_from_quick() {
let dict = FstDictionary::curated();
- assert_eq!(
+ assert!(
dict.get_word_metadata_str("quickly")
.unwrap()
.derived_from
- .unwrap(),
- WordId::from_word_str("quick")
+ .contains(CanonicalWordId::from_word_str("quick"))
);
}
diff --git a/harper-core/src/spell/merged_dictionary.rs b/harper-core/src/spell/merged_dictionary.rs
index 7e5e731d9..b3e3a56a8 100644
--- a/harper-core/src/spell/merged_dictionary.rs
+++ b/harper-core/src/spell/merged_dictionary.rs
@@ -5,7 +5,7 @@ use std::sync::Arc;
use foldhash::quality::FixedState;
use itertools::Itertools;
-use super::{FstDictionary, FuzzyMatchResult, WordId, dictionary::Dictionary};
+use super::{CanonicalWordId, FstDictionary, FuzzyMatchResult, dictionary::Dictionary};
use crate::{CharString, DictWordMetadata};
/// A simple wrapper over [`Dictionary`] that allows
@@ -136,6 +136,24 @@ impl Dictionary for MergedDictionary {
self.get_word_metadata(&chars)
}
+ fn get_word_metadata_exact(&self, word: &[char]) -> Option<&DictWordMetadata> {
+ self.children
+ .iter()
+ .find_map(|child| child.get_word_metadata_exact(word))
+ }
+
+ fn get_correct_capitalizations_of(
+ &self,
+ word: &[char],
+ ) -> Box<dyn Iterator<Item = &[char]> + Send + '_> {
+ let matches: Vec<_> = self
+ .children
+ .iter()
+ .map(|child| child.get_correct_capitalizations_of(word))
+ .collect();
+ Box::new(matches.into_iter().flatten())
+ }
+
fn fuzzy_match(
&'_ self,
word: &[char],
@@ -172,7 +190,7 @@ impl Dictionary for MergedDictionary {
self.children.iter().map(|d| d.word_count()).sum()
}
- fn get_word_from_id(&self, id: &WordId) -> Option<&[char]> {
+ fn get_word_from_id(&self, id: &CanonicalWordId) -> Option<&[char]> {
self.children
.iter()
.find_map(|dict| dict.get_word_from_id(id))
diff --git a/harper-core/src/spell/mod.rs b/harper-core/src/spell/mod.rs
index e4631e933..96ba0f26b 100644
--- a/harper-core/src/spell/mod.rs
+++ b/harper-core/src/spell/mod.rs
@@ -10,7 +10,7 @@ pub use self::fst_dictionary::FstDictionary;
pub use self::merged_dictionary::MergedDictionary;
pub use self::mutable_dictionary::MutableDictionary;
pub use self::trie_dictionary::TrieDictionary;
-pub use self::word_id::WordId;
+pub use self::word_id::{CanonicalWordId, CaseFoldedWordId};
mod dictionary;
mod fst_dictionary;
diff --git a/harper-core/src/spell/mutable_dictionary.rs b/harper-core/src/spell/mutable_dictionary.rs
index 32a35e024..0d4bc9263 100644
--- a/harper-core/src/spell/mutable_dictionary.rs
+++ b/harper-core/src/spell/mutable_dictionary.rs
@@ -1,7 +1,5 @@
use super::{
- FstDictionary, WordId,
- rune::{self, AttributeList, parse_word_list},
- word_map::{WordMap, WordMapEntry},
+ CanonicalWordId, CaseFoldedWordId, FstDictionary, FuzzyMatchResult, rune, word_map::WordMap,
};
use crate::edit_distance::edit_distance_min_alloc;
use itertools::Itertools;
@@ -10,7 +8,6 @@ use std::{borrow::Cow, sync::LazyLock};
use crate::{CharString, CharStringExt, DictWordMetadata};
-use super::FuzzyMatchResult;
use super::dictionary::Dictionary;
/// A basic dictionary that allows words to be added after instantiating.
@@ -31,8 +28,8 @@ pub struct MutableDictionary {
/// curated dictionary.
fn uncached_inner_new() -> Arc<MutableDictionary> {
MutableDictionary::from_rune_files(
- include_str!("../../dictionary.dict"),
- include_str!("../../annotations.json"),
+ super::dictionary::CURATED_DICT_STR,
+ super::dictionary::ANNOTATIONS_STR,
)
.map(Arc::new)
.unwrap_or_else(|e| panic!("Failed to load curated dictionary: {}", e))
@@ -48,15 +45,9 @@ impl MutableDictionary {
}
pub fn from_rune_files(word_list: &str, attr_list: &str) -> Result<Self, rune::Error> {
- let word_list = parse_word_list(word_list)?;
- let attr_list = AttributeList::parse(attr_list)?;
-
- // There will be at _least_ this number of words
- let mut word_map = WordMap::default();
-
- attr_list.expand_annotated_words(word_list, &mut word_map);
-
- Ok(Self { word_map })
+ Ok(Self {
+ word_map: WordMap::from_rune_files(word_list, attr_list)?,
+ })
}
/// Create a dictionary from the curated dictionary included
@@ -73,12 +64,7 @@ impl MutableDictionary {
&mut self,
words: impl IntoIterator<Item = (impl AsRef<[char]>, DictWordMetadata)>,
) {
- for (chars, metadata) in words.into_iter() {
- self.word_map.insert(WordMapEntry {
- metadata,
- canonical_spelling: chars.as_ref().into(),
- })
- }
+ self.word_map.extend_words(words);
}
/// Append a single word to the dictionary.
@@ -106,13 +92,24 @@ impl Default for MutableDictionary {
impl Dictionary for MutableDictionary {
fn get_word_metadata(&self, word: &[char]) -> Option<Cow<'_, DictWordMetadata>> {
- self.word_map
- .get_with_chars(word)
- .map(|v| Cow::Borrowed(&v.metadata))
+ let mut found_words = self.word_map.get_case_folded_chars(word);
+ let first = &found_words.next()?.metadata;
+
+ if let Some(second) = found_words.next() {
+ let mut first = first.clone();
+ first.merge(&second.metadata);
+ found_words.for_each(|entry| {
+ first.merge(&entry.metadata);
+ });
+ Some(Cow::Owned(first))
+ } else {
+ Some(Cow::Borrowed(first))
+ }
}
fn contains_word(&self, word: &[char]) -> bool {
- self.word_map.contains_chars(word)
+ self.word_map
+ .contains_case_folded(CaseFoldedWordId::from_word_chars(word).0)
}
fn contains_word_str(&self, word: &str) -> bool {
@@ -126,15 +123,32 @@ impl Dictionary for MutableDictionary {
}
fn get_correct_capitalization_of(&self, word: &[char]) -> Option<&'_ [char]> {
+ let mut entries = self.word_map.get_case_folded_chars(word);
+ let first = entries.next()?;
+ if entries.next().is_some() {
+ None
+ } else {
+ Some(first.canonical_spelling.as_slice())
+ }
+ }
+
+ fn get_correct_capitalizations_of(
+ &self,
+ word: &[char],
+ ) -> Box<dyn Iterator<Item = &[char]> + Send + '_> {
+ Box::new(
+ self.word_map
+ .get_case_folded_chars(word)
+ .map(|entry| entry.canonical_spelling.as_slice()),
+ )
+ }
+
+ fn get_word_metadata_exact(&self, word: &[char]) -> Option<&DictWordMetadata> {
self.word_map
- .get_with_chars(word)
- .map(|v| v.canonical_spelling.as_slice())
+ .get_canonical(CanonicalWordId::from_word_chars(word))
+ .map(|entry| &entry.metadata)
}
- /// Suggest a correct spelling for a given misspelled word.
- /// `Self::word` is assumed to be quite small (n < 100).
- /// `max_distance` relates to an optimization that allows the search
- /// algorithm to prune large portions of the search.
fn fuzzy_match(
&'_ self,
word: &[char],
@@ -185,7 +199,7 @@ impl Dictionary for MutableDictionary {
.map(|(word, edit_distance)| FuzzyMatchResult {
word,
edit_distance,
- metadata: self.get_word_metadata(word).unwrap(),
+ metadata: Cow::Borrowed(self.get_word_metadata_exact(word).unwrap()),
})
.collect()
}
@@ -213,15 +227,8 @@ impl Dictionary for MutableDictionary {
}
fn contains_exact_word(&self, word: &[char]) -> bool {
- let normalized = word.normalized();
-
- if let Some(found) = self.word_map.get_with_chars(normalized.as_ref())
- && found.canonical_spelling.as_ref() == normalized.as_ref()
- {
- return true;
- }
-
- false
+ self.word_map
+ .contains_canonical(CanonicalWordId::from_word_chars(word))
}
fn contains_exact_word_str(&self, word: &str) -> bool {
@@ -229,8 +236,10 @@ impl Dictionary for MutableDictionary {
self.contains_exact_word(word.as_ref())
}
- fn get_word_from_id(&self, id: &WordId) -> Option<&[char]> {
- self.word_map.get(id).map(|w| w.canonical_spelling.as_ref())
+ fn get_word_from_id(&self, id: &CanonicalWordId) -> Option<&[char]> {
+ self.word_map
+ .get_canonical(*id)
+ .map(|w| w.canonical_spelling.as_ref())
}
fn find_words_with_prefix(&self, prefix: &[char]) -> Vec<Cow<'_, [char]>> {
diff --git a/harper-core/src/spell/rune/attribute_list.rs b/harper-core/src/spell/rune/attribute_list.rs
index 1dcb85ea5..52879ba67 100644
--- a/harper-core/src/spell/rune/attribute_list.rs
+++ b/harper-core/src/spell/rune/attribute_list.rs
@@ -13,7 +13,7 @@ use super::expansion::{
};
use super::word_list::AnnotatedWord;
use crate::dict_word_metadata_orthography::OrthFlags;
-use crate::spell::WordId;
+use crate::spell::CanonicalWordId;
use crate::{CharString, DictWordMetadata, Span};
#[derive(Debug, Clone)]
@@ -64,6 +64,8 @@ impl AttributeList {
let orth_flags = OrthFlags::from_letters(&annotated_word.letters);
base_metadata.orth_info = orth_flags;
+ let annotated_word_id = CanonicalWordId::from_word_chars(&annotated_word.letters);
+
let mut conditional_expansion_metadata = Vec::new();
// First pass: Process all properties to build the base metadata
@@ -149,25 +151,25 @@ impl AttributeList {
word_map,
);
// Update the metadata of the expanded word
- let target_metadata = word_map.get_metadata_mut_chars(&new_word).unwrap();
+ let target_metadata = word_map
+ .get_metadata_mut_canonical(CanonicalWordId::from_word_chars(new_word))
+ .unwrap();
target_metadata.merge(&metadata);
- target_metadata.derived_from =
- Some(WordId::from_word_chars(&annotated_word.letters));
+ target_metadata.derived_from.insert(annotated_word_id);
}
} else {
// Simple case: no cross-product expansion needed
for (key, mut value) in new_words.into_iter() {
- value.derived_from = Some(WordId::from_word_chars(&annotated_word.letters));
+ value.derived_from.insert(annotated_word_id);
- if let Some(existing_metadata) = word_map.get_metadata_mut_chars(&key) {
+ if let Some(existing_metadata) =
+ word_map.get_metadata_mut_canonical(CanonicalWordId::from_word_chars(&key))
+ {
// Merge with existing metadata
existing_metadata.merge(&value);
} else {
// Add new entry
- word_map.insert(WordMapEntry {
- canonical_spelling: key,
- metadata: value,
- });
+ word_map.insert(WordMapEntry::new(key).with_md(value));
}
}
}
@@ -177,15 +179,12 @@ impl AttributeList {
let mut full_metadata = base_metadata;
// Merge with any existing metadata for this word
- if let Some(existing_metadata) = word_map.get_with_chars(&annotated_word.letters) {
- full_metadata.merge(&existing_metadata.metadata);
+ if let Some(existing_entry) = word_map.get_canonical(annotated_word_id) {
+ full_metadata.merge(&existing_entry.metadata);
}
// Store the final metadata for the base word
- word_map.insert(WordMapEntry {
- metadata: full_metadata.clone(),
- canonical_spelling: annotated_word.letters,
- });
+ word_map.insert(WordMapEntry::new(annotated_word.letters).with_md(full_metadata.clone()));
// Process any conditional expansions
for (letters, metadata, condition) in conditional_expansion_metadata {
@@ -197,7 +196,7 @@ impl AttributeList {
// Apply the conditional metadata
word_map
- .get_metadata_mut_chars(&letters)
+ .get_metadata_mut_canonical(CanonicalWordId::from_word_chars(letters))
.unwrap()
.merge(&metadata);
}
diff --git a/harper-core/src/spell/rune/mod.rs b/harper-core/src/spell/rune/mod.rs
index 3dcdaa258..af13e5dd7 100644
--- a/harper-core/src/spell/rune/mod.rs
+++ b/harper-core/src/spell/rune/mod.rs
@@ -19,6 +19,7 @@ mod tests {
use super::super::word_map::WordMap;
use super::word_list::parse_word_list;
use crate::CharStringExt;
+ use crate::spell::CanonicalWordId;
use crate::spell::rune::AttributeList;
pub const TEST_WORD_LIST: &str = "4\nhello\ntry/B\nwork/AB\nblank/";
@@ -208,10 +209,14 @@ mod tests {
attributes.expand_annotated_words(words, &mut expanded);
- let giant_data = expanded.get_with_str("giant").unwrap();
+ let giant_data = expanded
+ .get_canonical(CanonicalWordId::from_word_str("giant"))
+ .unwrap();
assert!(giant_data.metadata.is_noun());
- let giants_data = expanded.get_with_str("giants").unwrap();
+ let giants_data = expanded
+ .get_canonical(CanonicalWordId::from_word_str("giants"))
+ .unwrap();
assert!(giants_data.metadata.is_plural_noun());
}
}
diff --git a/harper-core/src/spell/trie_dictionary.rs b/harper-core/src/spell/trie_dictionary.rs
index a8b582db7..660ab8814 100644
--- a/harper-core/src/spell/trie_dictionary.rs
+++ b/harper-core/src/spell/trie_dictionary.rs
@@ -6,7 +6,7 @@ use trie_rs::iter::{Keys, PrefixIter, SearchIter};
use crate::DictWordMetadata;
-use super::{Dictionary, FstDictionary, FuzzyMatchResult, WordId};
+use super::{CanonicalWordId, Dictionary, FstDictionary, FuzzyMatchResult};
/// A [`Dictionary`] optimized for pre- and postfix search.
/// Wraps another dictionary to implement other operations.
@@ -89,10 +89,21 @@ impl<D: Dictionary> Dictionary for TrieDictionary<D> {
self.inner.word_count()
}
- fn get_word_from_id(&self, id: &WordId) -> Option<&[char]> {
+ fn get_word_from_id(&self, id: &CanonicalWordId) -> Option<&[char]> {
self.inner.get_word_from_id(id)
}
+ fn get_word_metadata_exact(&self, word: &[char]) -> Option<&DictWordMetadata> {
+ self.inner.get_word_metadata_exact(word)
+ }
+
+ fn get_correct_capitalizations_of(
+ &self,
+ word: &[char],
+ ) -> Box<dyn Iterator<Item = &[char]> + Send + '_> {
+ self.inner.get_correct_capitalizations_of(word)
+ }
+
fn find_words_with_prefix(&self, prefix: &[char]) -> Vec<Cow<'_, [char]>> {
let results: Keys<SearchIter<'_, char, (), Vec<char>, _>> =
self.trie.predictive_search(prefix);
diff --git a/harper-core/src/spell/word_id.rs b/harper-core/src/spell/word_id.rs
deleted file mode 100644
index f93fc2087..000000000
--- a/harper-core/src/spell/word_id.rs
+++ /dev/null
@@ -1,46 +0,0 @@
-use std::hash::BuildHasher;
-
-use foldhash::fast::FixedState;
-use serde::{Deserialize, Serialize};
-
-use crate::{CharString, CharStringExt};
-
-/// An identifier for a particular word.
-///
-/// It works by hashing the word it represents, normalized to lowercase.
-/// It is meant for situations where you need to refer to a word (or a collection of words),
-/// without storing all of accompanying data (like spelling or metadata).
-#[derive(Hash, Copy, Clone, PartialEq, Eq, PartialOrd, Debug, Serialize, Deserialize)]
-pub struct WordId {
- hash: u64,
-}
-
-impl WordId {
- /// Create a Word ID from a character slice.
- pub fn from_word_chars(chars: impl AsRef<[char]>) -> Self {
- let normalized = chars.as_ref().normalized();
- let lower = normalized.to_lower();
- let hash = FixedState::default().hash_one(lower);
-
- Self { hash }
- }
-
- /// Create a word ID from a string.
- /// Requires allocation, so use sparingly.
- pub fn from_word_str(text: impl AsRef<str>) -> Self {
- let chars: CharString = text.as_ref().chars().collect();
- Self::from_word_chars(chars)
- }
-}
-
-impl From<WordId> for u64 {
- fn from(val: WordId) -> Self {
- val.hash
- }
-}
-
-impl From<u64> for WordId {
- fn from(value: u64) -> Self {
- Self { hash: value }
- }
-}
diff --git a/harper-core/src/spell/word_id/canonical_word_id.rs b/harper-core/src/spell/word_id/canonical_word_id.rs
new file mode 100644
index 000000000..33eef03b4
--- /dev/null
+++ b/harper-core/src/spell/word_id/canonical_word_id.rs
@@ -0,0 +1,51 @@
+use std::hash::BuildHasher;
+
+use foldhash::fast::FixedState;
+use serde::{Deserialize, Serialize};
+
+use super::CaseFoldedWordId;
+use crate::CharString;
+
+/// An identifier for a particular word with canonical casing.
+#[derive(Hash, Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Debug, Serialize, Deserialize)]
+pub struct CanonicalWordId {
+ pub hash: u64,
+}
+
+impl CanonicalWordId {
+ /// Create a Word ID from a character slice.
+ pub fn from_word_chars(chars: impl AsRef<[char]>) -> Self {
+ let hash = FixedState::default().hash_one(chars.as_ref());
+
+ Self { hash }
+ }
+
+ /// Create a word ID from a string.
+ /// Requires allocation, so use sparingly.
+ pub fn from_word_str(text: impl AsRef<str>) -> Self {
+ let chars: CharString = text.as_ref().chars().collect();
+ Self::from_word_chars(chars)
+ }
+
+ /// Reinterpret this ID as a [`CaseFoldedWordId`].
+ ///
+ /// Note that this is just a reinterpretation, it does not perform any conversion. This is
+ /// useful when the canonical word ID is the same as the case-folded word ID. This will only
+ /// happen if the canonical ID was generated with a word that was already lowercased and
+ /// normalized.
+ pub fn as_case_folded(self) -> CaseFoldedWordId {
+ CaseFoldedWordId { hash: self.hash }
+ }
+}
+
+impl From<CanonicalWordId> for u64 {
+ fn from(val: CanonicalWordId) -> Self {
+ val.hash
+ }
+}
+
+impl From<u64> for CanonicalWordId {
+ fn from(value: u64) -> Self {
+ Self { hash: value }
+ }
+}
diff --git a/harper-core/src/spell/word_id/case_folded_word_id.rs b/harper-core/src/spell/word_id/case_folded_word_id.rs
new file mode 100644
index 000000000..393c1d8be
--- /dev/null
+++ b/harper-core/src/spell/word_id/case_folded_word_id.rs
@@ -0,0 +1,64 @@
+use std::{borrow::Cow, hash::BuildHasher};
+
+use foldhash::fast::FixedState;
+use serde::{Deserialize, Serialize};
+
+use crate::{CharString, CharStringExt, spell::CanonicalWordId};
+
+/// An identifier for a particular word with case-folded casing.
+///
+/// This does not usually point to a specific word, but rather a group of words that are identical
+/// when lowercased.
+#[derive(Hash, Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Debug, Serialize, Deserialize)]
+pub struct CaseFoldedWordId {
+ pub hash: u64,
+}
+
+impl CaseFoldedWordId {
+ /// Create a Word ID from a character slice.
+ ///
+ /// This will Case-fold and normalize the input before calculating the word ID.
+ ///
+ /// If the input word was already case-folded, this will also return
+ /// a [`CanonicalWordId`]. This is because the IDs will be identical under the hood, and the
+ /// canonical ID can be produced with no additional cost.
+ pub fn from_word_chars(chars: impl AsRef<[char]>) -> (Self, Option<CanonicalWordId>) {
+ let normalized = chars.as_ref().normalized();
+ let lower = normalized.to_lower();
+
+ let was_already_case_folded =
+ matches!(normalized, Cow::Borrowed(_)) && matches!(lower, Cow::Borrowed(_));
+
+ let hash = FixedState::default().hash_one(lower);
+
+ (
+ Self { hash },
+ was_already_case_folded.then_some(CanonicalWordId { hash }),
+ )
+ }
+
+ /// Create a word ID from a string.
+ /// Requires allocation, so use sparingly.
+ ///
+ /// This will case-fold and normalize the input before calculating the word ID.
+ ///
+ /// If the input word was already case-folded, this will also return
+ /// a [`CanonicalWordId`]. This is because the IDs will be identical under the hood, and the
+ /// canonical ID can be produced with no additional cost.
+ pub fn from_word_str(text: impl AsRef<str>) -> (Self, Option<CanonicalWordId>) {
+ let chars: CharString = text.as_ref().chars().collect();
+ Self::from_word_chars(chars)
+ }
+}
+
+impl From<CaseFoldedWordId> for u64 {
+ fn from(val: CaseFoldedWordId) -> Self {
+ val.hash
+ }
+}
+
+impl From<u64> for CaseFoldedWordId {
+ fn from(value: u64) -> Self {
+ Self { hash: value }
+ }
+}
diff --git a/harper-core/src/spell/word_id/mod.rs b/harper-core/src/spell/word_id/mod.rs
new file mode 100644
index 000000000..37f8ed4b2
--- /dev/null
+++ b/harper-core/src/spell/word_id/mod.rs
@@ -0,0 +1,12 @@
+//! Identifiers for words.
+//!
+//! These are meant for situations where you need to refer to a word (or a collection of words),
+//! without storing all of accompanying data (like spelling or metadata).
+
+pub use canonical_word_id::CanonicalWordId;
+pub use case_folded_word_id::CaseFoldedWordId;
+pub use word_id_pair::WordIdPair;
+
+mod canonical_word_id;
+mod case_folded_word_id;
+mod word_id_pair;
diff --git a/harper-core/src/spell/word_id/word_id_pair.rs b/harper-core/src/spell/word_id/word_id_pair.rs
new file mode 100644
index 000000000..523ed2742
--- /dev/null
+++ b/harper-core/src/spell/word_id/word_id_pair.rs
@@ -0,0 +1,45 @@
+use serde::{Deserialize, Serialize};
+
+use super::{CanonicalWordId, CaseFoldedWordId};
+use crate::CharString;
+
+/// A pair containing both [`CanonicalWordId`] and [`CaseFoldedWordId`] for a given word.
+#[derive(Hash, Copy, Clone, PartialEq, Eq, PartialOrd, Debug, Serialize, Deserialize)]
+pub struct WordIdPair {
+ pub canonical: CanonicalWordId,
+ pub case_folded: CaseFoldedWordId,
+}
+
+impl WordIdPair {
+ /// Create a Word ID pair from a character slice.
+ ///
+ /// Calculates both the canonical and case-folded word ID for the provided word.
+ pub fn from_word_chars(chars: impl AsRef<[char]>) -> Self {
+ let (case_folded, maybe_canonical) = CaseFoldedWordId::from_word_chars(chars.as_ref());
+
+ Self {
+ // Trying to avoid hashing again if possible.
+ canonical: maybe_canonical.unwrap_or_else(|| CanonicalWordId::from_word_chars(chars)),
+ case_folded,
+ }
+ }
+
+ /// Create a word ID pair from a string.
+ /// Requires allocation, so use sparingly.
+ ///
+ /// Calculates both the canonical and case-folded word ID for the provided word.
+ pub fn from_word_str(text: impl AsRef<str>) -> Self {
+ let chars: CharString = text.as_ref().chars().collect();
+ Self::from_word_chars(chars)
+ }
+
+ /// The canonical ID of the word.
+ pub fn canonical(&self) -> CanonicalWordId {
+ self.canonical
+ }
+
+ /// The case-folded ID of the word.
+ pub fn case_folded(&self) -> CaseFoldedWordId {
+ self.case_folded
+ }
+}
diff --git a/harper-core/src/spell/word_map.rs b/harper-core/src/spell/word_map.rs
deleted file mode 100644
index 8f97ec884..000000000
--- a/harper-core/src/spell/word_map.rs
+++ /dev/null
@@ -1,104 +0,0 @@
-use hashbrown::{HashMap, hash_map::IntoValues};
-
-use crate::{CharString, DictWordMetadata};
-
-use super::WordId;
-
-/// The underlying data structure for the `MutableDictionary`.
-#[derive(Debug, Clone, Eq, PartialEq, Default)]
-pub struct WordMap {
- inner: HashMap<WordId, WordMapEntry>,
-}
-
-#[derive(Debug, Clone, Eq, PartialEq, Hash)]
-pub struct WordMapEntry {
- pub metadata: DictWordMetadata,
- pub canonical_spelling: CharString,
-}
-
-impl WordMap {
- /// Get an entry from the word map using raw chars.
- pub fn get_with_str(&self, string: &str) -> Option<&WordMapEntry> {
- let chars: CharString = string.chars().collect();
- let id = WordId::from_word_chars(chars);
-
- self.get(&id)
- }
-
- pub fn contains_str(&self, string: &str) -> bool {
- self.get_with_str(string).is_some()
- }
-
- pub fn contains_chars(&self, chars: impl AsRef<[char]>) -> bool {
- self.get_with_chars(chars).is_some()
- }
-
- pub fn contains(&self, id: &WordId) -> bool {
- self.get(id).is_some()
- }
-
- /// Get an entry from the word map using raw chars.
- pub fn get_with_chars(&self, chars: impl AsRef<[char]>) -> Option<&WordMapEntry> {
- let id = WordId::from_word_chars(chars);
-
- self.get(&id)
- }
-
- /// Get an entry from the word map using a word identifier.
- pub fn get(&self, id: &WordId) -> Option<&WordMapEntry> {
- self.inner.get(id)
- }
-
- /// Borrow a word's metadata mutably
- pub fn get_metadata_mut_chars(
- &mut self,
- chars: impl AsRef<[char]>,
- ) -> Option<&mut DictWordMetadata> {
- let id = WordId::from_word_chars(chars);
-
- self.get_metadata_mut(&id)
- }
-
- /// Borrow a word's metadata mutably
- pub fn get_metadata_mut(&mut self, id: &WordId) -> Option<&mut DictWordMetadata> {
- self.inner.get_mut(id).map(|v| &mut v.metadata)
- }
-
- pub fn insert(&mut self, entry: WordMapEntry) {
- let id = WordId::from_word_chars(&entry.canonical_spelling);
-
- self.inner.insert(id, entry);
- }
-
- /// Reserves capacity for at least `additional` more elements to be inserted
- /// in the `WordMap`. The collection may reserve more space to avoid
- /// frequent reallocations.
- pub fn reserve(&mut self, additional: usize) {
- self.inner.reserve(additional);
- }
-
- /// Iterate through the canonical spellings of the words in the map.
- pub fn iter(&self) -> impl Iterator<Item = &WordMapEntry> {
- self.inner.values()
- }
-
- pub fn len(&self) -> usize {
- self.inner.len()
- }
-
- pub fn with_capacity(capacity: usize) -> Self {
- Self {
- inner: HashMap::with_capacity(capacity),
- }
- }
-}
-
-impl IntoIterator for WordMap {
- type Item = WordMapEntry;
-
- fn into_iter(self) -> Self::IntoIter {
- self.inner.into_values()
- }
-
- type IntoIter = IntoValues<WordId, WordMapEntry>;
-}
diff --git a/harper-core/src/spell/word_map/mod.rs b/harper-core/src/spell/word_map/mod.rs
new file mode 100644
index 000000000..e080dd643
--- /dev/null
+++ b/harper-core/src/spell/word_map/mod.rs
@@ -0,0 +1,252 @@
+use std::{
+ ops::Index,
+ sync::{Arc, LazyLock},
+};
+
+use hashbrown::{DefaultHashBuilder, HashMap};
+use indexmap::IndexMap;
+
+use crate::{
+ DictWordMetadata,
+ spell::{
+ dictionary::{ANNOTATIONS_STR, CURATED_DICT_STR},
+ rune::{self, AttributeList, parse_word_list},
+ word_id::{CanonicalWordId, CaseFoldedWordId},
+ },
+};
+
+pub mod word_map_entry;
+pub use word_map_entry::WordMapEntry;
+
+/// Type used to store the canonical entries in the word map.
+///
+/// This currently uses an [`IndexMap`], so a word can be indexed either by its canonical word ID
+/// (which is the same across all word maps), or directly via a `usize` index (specific to this
+/// word map only).
+///
+/// The latter property is used to allow [`WordMap::case_folded`] to indirectly reference entries
+/// in [`WordMap::canonical`].
+type CanonicalStorage = IndexMap<CanonicalWordId, WordMapEntry, DefaultHashBuilder>;
+
+/// The most basic dictionary. Stores a list of words and their metadata, while allowing for
+/// efficient case-folded lookups.
+///
+/// This is the underlying data structure for all other dictionaries (at the time of writing).
+#[derive(Debug, Clone, Eq, PartialEq, Default)]
+pub struct WordMap {
+ /// Underlying container for the entries in the word map.
+ canonical: CanonicalStorage,
+ /// A map containing indices into `canonical` for a specific `CaseFoldedWordId`. This is used for
+ /// case-folded lookups in the word map.
+ case_folded: HashMap<CaseFoldedWordId, Vec<usize>>,
+}
+
+impl WordMap {
+ /// Create an empty word map.
+ pub fn new() -> Self {
+ Default::default()
+ }
+
+ /// A word map containing entries from the curated dictionary.
+ pub fn curated() -> Arc<Self> {
+ /// A word map containing entries from the curated dictionary.
+ static CURATED: LazyLock<Arc<WordMap>> = LazyLock::new(|| {
+ Arc::new(WordMap::from_rune_files(CURATED_DICT_STR, ANNOTATIONS_STR).unwrap())
+ });
+
+ (*CURATED).clone()
+ }
+
+ /// Does the word map contain a word with the given [`CanonicalWordId`]?
+ pub fn contains_canonical(&self, id: CanonicalWordId) -> bool {
+ self.get_canonical(id).is_some()
+ }
+
+ /// Does the word map contain any words with the given [`CaseFoldedWordId`]?
+ pub fn contains_case_folded(&self, id: CaseFoldedWordId) -> bool {
+ !self.get_canonical_indices_from_case_folded(id).is_empty()
+ }
+
+ /// Get an entry from the word map using a word identifier.
+ pub fn get_canonical(&self, id: CanonicalWordId) -> Option<&WordMapEntry> {
+ self.canonical.get(&id)
+ }
+
+ /// Get the entries corresponding to the provided [`CaseFoldedWordId`].
+ pub fn get_case_folded(
+ &self,
+ id: CaseFoldedWordId,
+ ) -> impl ExactSizeIterator<Item = &WordMapEntry> {
+ self.get_canonical_indices_from_case_folded(id)
+ .iter()
+ .map(|canonical_index| &self[*canonical_index])
+ }
+
+ /// Convenience wrapper for [`Self::get_case_folded`].
+ ///
+ /// This recalculates the word ID every time. If you're going to be querying the same word
+ /// multiple times, consider storing the word ID and using [`Self::get_case_folded`] instead.
+ pub fn get_case_folded_chars(
+ &self,
+ word: &[char],
+ ) -> impl ExactSizeIterator<Item = &WordMapEntry> + use<'_> {
+ self.get_case_folded(CaseFoldedWordId::from_word_chars(word).0)
+ }
+
+ /// Borrow a word's metadata mutably
+ pub fn get_metadata_mut_canonical(
+ &mut self,
+ id: CanonicalWordId,
+ ) -> Option<&mut DictWordMetadata> {
+ self.canonical.get_mut(&id).map(|v| &mut v.metadata)
+ }
+
+ /// Insert an entry into the word map.
+ ///
+ /// This will merge the new metadata with the existing if an entry with an identical
+ /// [`CanonicalWordId`] is already contained in the word map.
+ ///
+ /// If you are appending many words, consider using [`Self::extend`] instead.
+ pub fn insert(&mut self, entry: WordMapEntry) {
+ let (canonical_id, case_folded_id) =
+ (entry.word_ids.canonical(), entry.word_ids.case_folded());
+
+ if let Some(existing_entry) = self.canonical.get_mut(&canonical_id) {
+ // An existing word with the same canonical ID exists; update its entry.
+ existing_entry.metadata.merge(&entry.metadata);
+ } else {
+ // An existing word with the same canonical ID does NOT exist; insert it.
+ let (canonical_idx, _) = self.canonical.insert_full(canonical_id, entry);
+ if let Some(existing_case_folded_entry) = self.case_folded.get_mut(&case_folded_id) {
+ // `case_folded` already has a canonical ID list for this word; append to it, if
+ // the same entry does not already exist.
+ if !existing_case_folded_entry.contains(&canonical_idx) {
+ existing_case_folded_entry.push(canonical_idx);
+ }
+ } else {
+ // `case_folded` does NOT have a canonical ID list for this word; initialize one.
+ self.case_folded.insert(case_folded_id, vec![canonical_idx]);
+ }
+ }
+ }
+
+ /// Reserves capacity for at least `additional` more elements to be inserted
+ /// in the `WordMap`. The collection may reserve more space to avoid
+ /// frequent reallocations.
+ pub fn reserve(&mut self, additional: usize) {
+ self.canonical.reserve(additional);
+ }
+
+ /// Iterate through the canonical spellings of the words in the map.
+ pub fn iter(&self) -> impl ExactSizeIterator<Item = &WordMapEntry> {
+ self.canonical.values()
+ }
+
+ /// The number of words in the word map.
+ pub fn len(&self) -> usize {
+ self.canonical.len()
+ }
+
+ pub fn is_empty(&self) -> bool {
+ self.len() == 0
+ }
+
+ pub fn with_capacity(capacity: usize) -> Self {
+ Self {
+ canonical: IndexMap::with_capacity_and_hasher(capacity, DefaultHashBuilder::default()),
+ case_folded: HashMap::new(),
+ }
+ }
+
+ pub fn from_rune_files(word_list: &str, attr_list: &str) -> Result<Self, rune::Error> {
+ let word_list = parse_word_list(word_list)?;
+ let attr_list = AttributeList::parse(attr_list)?;
+
+ let mut word_map = WordMap::default();
+
+ attr_list.expand_annotated_words(word_list, &mut word_map);
+
+ Ok(word_map)
+ }
+
+ /// Get indices into [`Self::canonical`] using the provided [`CaseFoldedWordId`].
+ fn get_canonical_indices_from_case_folded(&self, id: CaseFoldedWordId) -> &[usize] {
+ self.case_folded
+ .get(&id)
+ .map_or(&[], |canonical_indices| canonical_indices)
+ }
+}
+
+// Functions for backward compatibility.
+impl WordMap {
+ /// Append a single word to the dictionary.
+ ///
+ /// If you are appending many words, consider using [`Self::extend_words`]
+ /// instead.
+ pub fn append_word(&mut self, word: impl AsRef<[char]>, metadata: DictWordMetadata) {
+ self.insert(WordMapEntry::new(word.as_ref()).with_md(metadata));
+ }
+
+ /// Append a single word to the dictionary.
+ ///
+ /// If you are appending many words, consider using [`Self::extend_words`]
+ /// instead.
+ pub fn append_word_str(&mut self, word: &str, metadata: DictWordMetadata) {
+ self.insert(WordMapEntry::new_str(word).with_md(metadata));
+ }
+
+ /// Appends words to the dictionary.
+ pub fn extend_words(
+ &mut self,
+ words: impl IntoIterator<Item = (impl AsRef<[char]>, DictWordMetadata)>,
+ ) {
+ self.extend(words.into_iter().map(|(canonical_spelling, metadata)| {
+ WordMapEntry::new(canonical_spelling.as_ref()).with_md(metadata)
+ }))
+ }
+}
+
+impl Extend<WordMapEntry> for WordMap {
+ fn extend<T: IntoIterator<Item = WordMapEntry>>(&mut self, iter: T) {
+ // [Copied from HashBrown::map::HashMap::extend]
+ // Keys may be already present or show multiple times in the iterator.
+ // Reserve the entire hint lower bound if the map is empty.
+ // Otherwise reserve half the hint (rounded up), so the map
+ // will only resize twice in the worst case.
+ let iter = iter.into_iter();
+ let reserve = if self.is_empty() {
+ iter.size_hint().0
+ } else {
+ iter.size_hint().0.div_ceil(2)
+ };
+ self.reserve(reserve);
+ iter.for_each(move |wme| {
+ self.insert(wme);
+ });
+ }
+}
+
+impl Index<usize> for WordMap {
+ type Output = <CanonicalStorage as Index<usize>>::Output;
+
+ fn index(&self, index: usize) -> &Self::Output {
+ &self.canonical[index]
+ }
+}
+
+impl IntoIterator for WordMap {
+ type Item = WordMapEntry;
+ type IntoIter = indexmap::map::IntoValues<CanonicalWordId, WordMapEntry>;
+
+ fn into_iter(self) -> Self::IntoIter {
+ self.canonical.into_values()
+ }
+}
+
+impl FromIterator<WordMapEntry> for WordMap {
+ fn from_iter<T: IntoIterator<Item = WordMapEntry>>(iter: T) -> Self {
+ let mut out = Self::new();
+ out.extend(iter);
+ out
+ }
+}
diff --git a/harper-core/src/spell/word_map/word_map_entry.rs b/harper-core/src/spell/word_map/word_map_entry.rs
new file mode 100644
index 000000000..047ec195a
--- /dev/null
+++ b/harper-core/src/spell/word_map/word_map_entry.rs
@@ -0,0 +1,35 @@
+use crate::{CharString, DictWordMetadata, spell::word_id::WordIdPair};
+
+#[derive(Debug, Clone, Eq, PartialEq, Hash)]
+pub struct WordMapEntry {
+ pub metadata: DictWordMetadata,
+ pub canonical_spelling: CharString,
+ pub word_ids: WordIdPair,
+}
+
+impl WordMapEntry {
+ /// Construct a new word map entry with default metadata.
+ pub fn new(canonical_spelling: impl Into<CharString>) -> Self {
+ Self::from_chars(canonical_spelling.into())
+ }
+
+ /// Construct a new word map entry with default metadata.
+ pub fn new_str(canonical_spelling: &str) -> Self {
+ Self::from_chars(canonical_spelling.chars().collect())
+ }
+
+ fn from_chars(canonical_spelling: CharString) -> Self {
+ let word_ids = WordIdPair::from_word_chars(&canonical_spelling);
+
+ Self {
+ metadata: Default::default(),
+ canonical_spelling,
+ word_ids,
+ }
+ }
+
+ pub fn with_md(mut self, metadata: DictWordMetadata) -> Self {
+ self.metadata = metadata;
+ self
+ }
+}
diff --git a/harper-core/tests/run_tests.rs b/harper-core/tests/run_tests.rs
index 9341420ca..e6ef8c9f6 100644
--- a/harper-core/tests/run_tests.rs
+++ b/harper-core/tests/run_tests.rs
@@ -79,7 +79,7 @@ create_test!(issue_159.md, 1, Dialect::American);
create_test!(issue_358.md, 0, Dialect::American);
create_test!(issue_195.md, 0, Dialect::American);
create_test!(issue_118.md, 0, Dialect::American);
-create_test!(lots_of_latin.md, 1, Dialect::American);
+create_test!(lots_of_latin.md, 0, Dialect::American);
create_test!(pr_504.md, 1, Dialect::American);
create_test!(pr_452.md, 2, Dialect::American);
create_test!(hex_basic_clean.md, 0, Dialect::American);
@@ -105,4 +105,4 @@ create_test!(issue_2151.md, 3, Dialect::British);
create_test!(lukas_homework.md, 4, Dialect::American);
// Org mode tests
-create_org_test!(index.org, 49, Dialect::American);
+create_org_test!(index.org, 48, Dialect::American);
diff --git a/harper-core/tests/text/linters/Alice's Adventures in Wonderland.snap.yml b/harper-core/tests/text/linters/Alice's Adventures in Wonderland.snap.yml
index bf2c14e85..681b36576 100644
--- a/harper-core/tests/text/linters/Alice's Adventures in Wonderland.snap.yml
+++ b/harper-core/tests/text/linters/Alice's Adventures in Wonderland.snap.yml
@@ -625,26 +625,6 @@ Suggest:
-Lint: Capitalization (127 priority)
-Message: |
- 320 | she began again: “Où est ma chatte?” which was the first sentence in her French
- | ^~~ This word's canonical spelling is all-caps.
-Suggest:
- - Replace with: “EST”
-
-
-
-Lint: Spelling (63 priority)
-Message: |
- 320 | she began again: “Où est ma chatte?” which was the first sentence in her French
- | ^~~ Did you mean to spell `est` this way?
-Suggest:
- - Replace with: “east”
- - Replace with: “eat”
- - Replace with: “esp”
-
-
-
Lint: Spelling (63 priority)
Message: |
320 | she began again: “Où est ma chatte?” which was the first sentence in her French
@@ -998,15 +978,6 @@ Message: |
-Lint: Capitalization (31 priority)
-Message: |
- 607 | Alas! it was too late to wish that! She went on growing, and growing, and very
- | ^~ This sentence does not start with a capital letter
-Suggest:
- - Replace with: “It”
-
-
-
Lint: Readability (127 priority)
Message: |
607 | Alas! it was too late to wish that! She went on growing, and growing, and very
@@ -1454,15 +1425,6 @@ Suggest:
-Lint: Capitalization (127 priority)
-Message: |
- 1059 | ## CHAPTER VI: Pig and Pepper
- | ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Try to use title case in headings.
-Suggest:
- - Replace with: “## CHAPTER Vi: Pig and Pepper”
-
-
-
Lint: Readability (127 priority)
Message: |
1061 | For a minute or two she stood looking at the house, and wondering what to do
diff --git a/harper-core/tests/text/linters/Computer science.snap.yml b/harper-core/tests/text/linters/Computer science.snap.yml
index c672cd12c..7d7e902e7 100644
--- a/harper-core/tests/text/linters/Computer science.snap.yml
+++ b/harper-core/tests/text/linters/Computer science.snap.yml
@@ -46,8 +46,8 @@ Message: |
| ^~~~~~~~ Did you mean to spell `Reckoner` this way?
Suggest:
- Replace with: “Reckoned”
- - Replace with: “Rickover”
- Replace with: “Reasoner”
+ - Replace with: “Rickover”
@@ -598,7 +598,7 @@ Message: |
Suggest:
- Replace with: “Paras”
- Replace with: “Parkas”
- - Replace with: “Parana's”
+ - Replace with: “Pan's”
@@ -879,17 +879,6 @@ Suggest:
-Lint: Spelling (63 priority)
-Message: |
- 262 | The famous P = NP? problem, one of the Millennium Prize Problems, is an open
- | ^~ Did you mean to spell `NP` this way?
-Suggest:
- - Replace with: “Nap”
- - Replace with: “Nip”
- - Replace with: “No”
-
-
-
Lint: Capitalization (31 priority)
Message: |
262 | The famous P = NP? problem, one of the Millennium Prize Problems, is an open
@@ -1051,8 +1040,8 @@ Message: |
| ^~~ Did you mean to spell `HCI` this way?
Suggest:
- Replace with: “Hi”
- - Replace with: “H'm”
- Replace with: “Chi”
+ - Replace with: “Sci”
@@ -1063,8 +1052,8 @@ Message: |
349 | that focus on the relationship between emotions, social behavior and brain
Suggest:
- Replace with: “Hi”
- - Replace with: “H'm”
- Replace with: “Chi”
+ - Replace with: “Sci”
@@ -1120,7 +1109,7 @@ Message: |
Suggest:
- Replace with: “RI”
- Replace with: “Ra”
- - Replace with: “Ru”
+ - Replace with: “Re”
@@ -1321,9 +1310,9 @@ Message: |
469 | goto (which means it is more elementary than structured programming).
| ^~~~ Did you mean to spell `goto` this way?
Suggest:
+ - Replace with: “goth”
- Replace with: “goo”
- Replace with: “got”
- - Replace with: “goths”
diff --git a/harper-core/tests/text/linters/Part-of-speech tagging.snap.yml b/harper-core/tests/text/linters/Part-of-speech tagging.snap.yml
index d3a9466db..e49b84812 100644
--- a/harper-core/tests/text/linters/Part-of-speech tagging.snap.yml
+++ b/harper-core/tests/text/linters/Part-of-speech tagging.snap.yml
@@ -59,8 +59,8 @@ Message: |
| ^~~~~~~ Did you mean to spell `Brill's` this way?
19 | first and most widely used English POS-taggers, employs rule-based algorithms.
Suggest:
- - Replace with: “Brillo's”
- Replace with: “Bill's”
+ - Replace with: “Brillo's”
- Replace with: “Drill's”
@@ -121,18 +121,6 @@ Suggest:
-Lint: Spelling (63 priority)
-Message: |
- 49 | tags. For example, NN for singular common nouns, NNS for plural common nouns, NP
- | ^~ Did you mean to spell `NP` this way?
- 50 | for singular proper nouns (see the POS tags used in the Brown Corpus). Other
-Suggest:
- - Replace with: “Nap”
- - Replace with: “Nip”
- - Replace with: “No”
-
-
-
Lint: Spelling (63 priority)
Message: |
55 | 150 separate parts of speech for English. Work on stochastic methods for tagging
@@ -605,7 +593,7 @@ Message: |
Suggest:
- Replace with: “Sim”
- Replace with: “Sum”
- - Replace with: “SCM”
+ - Replace with: “SAM”
diff --git a/harper-core/tests/text/linters/The Constitution of the United States.snap.yml b/harper-core/tests/text/linters/The Constitution of the United States.snap.yml
index bb4086ea5..b8fa8d8d5 100644
--- a/harper-core/tests/text/linters/The Constitution of the United States.snap.yml
+++ b/harper-core/tests/text/linters/The Constitution of the United States.snap.yml
@@ -1718,15 +1718,6 @@ Message: |
-Lint: Capitalization (127 priority)
-Message: |
- 677 | ## Article. VI.
- | ^~~~~~~~~~~~~~~ Try to use title case in headings.
-Suggest:
- - Replace with: “## Article. Vi.”
-
-
-
Lint: Readability (127 priority)
Message: |
683 | This Constitution, and the Laws of the United States which shall be made in
diff --git a/harper-core/tests/text/linters/The Great Gatsby.snap.yml b/harper-core/tests/text/linters/The Great Gatsby.snap.yml
index 6e36711de..8d6fc60cb 100644
--- a/harper-core/tests/text/linters/The Great Gatsby.snap.yml
+++ b/harper-core/tests/text/linters/The Great Gatsby.snap.yml
@@ -882,7 +882,7 @@ Message: |
Suggest:
- Replace with: “Be”
- Replace with: “Bi”
- - Replace with: “BO”
+ - Replace with: “BA”
@@ -967,7 +967,7 @@ Message: |
Suggest:
- Replace with: “Do”
- Replace with: “DA”
- - Replace with: “Di”
+ - Replace with: “DE”
@@ -1133,7 +1133,7 @@ Message: |
Suggest:
- Replace with: “Be”
- Replace with: “Bi”
- - Replace with: “BO”
+ - Replace with: “BA”
@@ -2215,7 +2215,7 @@ Message: |
Suggest:
- Replace with: “RI”
- Replace with: “Ra”
- - Replace with: “Ru”
+ - Replace with: “Re”
@@ -2266,8 +2266,8 @@ Message: |
1861 | the gravel drive that Mrs. Ulysses Swett’s automobile ran over his right hand.
| ^~~~~~~ Did you mean to spell `Swett’s` this way?
Suggest:
- - Replace with: “Sweat's”
- Replace with: “Sweet's”
+ - Replace with: “Sweat's”
- Replace with: “Seth's”
@@ -2290,7 +2290,7 @@ Message: |
Suggest:
- Replace with: “So”
- Replace with: “SA”
- - Replace with: “Se”
+ - Replace with: “SE”
@@ -2301,7 +2301,7 @@ Message: |
Suggest:
- Replace with: “Be”
- Replace with: “Bi”
- - Replace with: “BO”
+ - Replace with: “BA”
@@ -2394,7 +2394,7 @@ Message: |
Suggest:
- Replace with: “So”
- Replace with: “SA”
- - Replace with: “Se”
+ - Replace with: “SE”
@@ -2440,8 +2440,8 @@ Message: |
| ^~ Did you mean to spell `G.` this way?
Suggest:
- Replace with: “Go”
- - Replace with: “GI”
- - Replace with: “GU”
+ - Replace with: “GA”
+ - Replace with: “GE”
@@ -2517,7 +2517,7 @@ Message: |
Suggest:
- Replace with: “Lear's”
- Replace with: “Leger's”
- - Replace with: “Leer's”
+ - Replace with: “Lego's”
@@ -2529,7 +2529,7 @@ Message: |
Suggest:
- Replace with: “Be”
- Replace with: “Bi”
- - Replace with: “BO”
+ - Replace with: “BA”
@@ -2541,7 +2541,7 @@ Message: |
Suggest:
- Replace with: “Jon's”
- Replace with: “Jonas”
- - Replace with: “Jogs”
+ - Replace with: “Jones”
@@ -2692,7 +2692,7 @@ Message: |
Suggest:
- Replace with: “So”
- Replace with: “SA”
- - Replace with: “Se”
+ - Replace with: “SE”
@@ -2748,9 +2748,9 @@ Message: |
1882 | and Henry L. Palmetto, who killed himself by jumping in front of a subway train
| ^~ Did you mean to spell `L.` this way?
Suggest:
+ - Replace with: “LA”
- Replace with: “Le”
- Replace with: “Li”
- - Replace with: “Lu”
@@ -2924,8 +2924,8 @@ Message: |
1982 | de Boulogne.
Suggest:
- Replace with: “Boss”
+ - Replace with: “Boas”
- Replace with: “Boris”
- - Replace with: “Bios”
@@ -2962,16 +2962,6 @@ Suggest:
-Lint: Capitalization (127 priority)
-Message: |
- 2006 | To my astonishment, the thing had an authentic look. “Orderi di Danilo,” ran the
- | ^~ This word's canonical spelling is all-caps.
- 2007 | circular legend, “Montenegro, Nicolas Rex.”
-Suggest:
- - Replace with: “DI”
-
-
-
Lint: Spelling (63 priority)
Message: |
2006 | To my astonishment, the thing had an authentic look. “Orderi di Danilo,” ran the
@@ -4120,15 +4110,6 @@ Suggest:
-Lint: Capitalization (127 priority)
-Message: |
- 3003 | ## CHAPTER VI
- | ^~~~~~~~~~~~~ Try to use title case in headings.
-Suggest:
- - Replace with: “## CHAPTER Vi”
-
-
-
Lint: Readability (127 priority)
Message: |
3020 | short of being news. Contemporary legends such as the “underground pipe-line to
@@ -5165,7 +5146,7 @@ Message: |
Suggest:
- Replace with: “Biko's”
- Replace with: “Bilbo's”
- - Replace with: “Biro's”
+ - Replace with: “Bill's”
@@ -5176,7 +5157,7 @@ Message: |
Suggest:
- Replace with: “Biko's”
- Replace with: “Bilbo's”
- - Replace with: “Biro's”
+ - Replace with: “Bill's”
@@ -5187,7 +5168,7 @@ Message: |
Suggest:
- Replace with: “Biko's”
- Replace with: “Bilbo's”
- - Replace with: “Biro's”
+ - Replace with: “Bill's”
@@ -5199,7 +5180,7 @@ Message: |
Suggest:
- Replace with: “Biko's”
- Replace with: “Bilbo's”
- - Replace with: “Biro's”
+ - Replace with: “Bill's”
@@ -5217,7 +5198,7 @@ Message: |
Suggest:
- Replace with: “Biko's”
- Replace with: “Bilbo's”
- - Replace with: “Biro's”
+ - Replace with: “Bill's”
@@ -5270,7 +5251,7 @@ Message: |
Suggest:
- Replace with: “Biko's”
- Replace with: “Bilbo's”
- - Replace with: “Biro's”
+ - Replace with: “Bill's”
@@ -5288,7 +5269,7 @@ Message: |
Suggest:
- Replace with: “Biko's”
- Replace with: “Bilbo's”
- - Replace with: “Biro's”
+ - Replace with: “Bill's”
@@ -5311,7 +5292,7 @@ Message: |
Suggest:
- Replace with: “Biko's”
- Replace with: “Bilbo's”
- - Replace with: “Biro's”
+ - Replace with: “Bill's”
@@ -5322,7 +5303,7 @@ Message: |
Suggest:
- Replace with: “Biko's”
- Replace with: “Bilbo's”
- - Replace with: “Biro's”
+ - Replace with: “Bill's”
@@ -5758,8 +5739,8 @@ Message: |
| ^ Did you mean to spell `r` this way?
Suggest:
- Replace with: “re”
+ - Replace with: “I”
- Replace with: “a”
- - Replace with: “i”
@@ -5769,8 +5750,8 @@ Message: |
| ^ Did you mean to spell `r` this way?
Suggest:
- Replace with: “re”
+ - Replace with: “I”
- Replace with: “a”
- - Replace with: “i”
@@ -5800,8 +5781,8 @@ Message: |
| ^ Did you mean to spell `r` this way?
Suggest:
- Replace with: “re”
+ - Replace with: “I”
- Replace with: “a”
- - Replace with: “i”
@@ -6638,9 +6619,9 @@ Message: |
5172 | Roosevelt and then to Gad’s Hill, where he bought a sandwich that he didn’t eat,
| ^~~~~ Did you mean to spell `Gad’s` this way?
Suggest:
- - Replace with: “Gab's”
- - Replace with: “Gag's”
- - Replace with: “Gal's”
+ - Replace with: “Gap's”
+ - Replace with: “Gay's”
+ - Replace with: “God's”
@@ -6650,9 +6631,9 @@ Message: |
5174 | reach Gad’s Hill until noon. Thus far there was no difficulty in accounting for
| ^~~~~ Did you mean to spell `Gad’s` this way?
Suggest:
- - Replace with: “Gab's”
- - Replace with: “Gag's”
- - Replace with: “Gal's”
+ - Replace with: “Gap's”
+ - Replace with: “Gay's”
+ - Replace with: “God's”
@@ -6917,9 +6898,9 @@ Message: |
5341 | “Young Parke’s in trouble,” he said rapidly. “They picked him up when he handed
| ^~~~~~~ Did you mean to spell `Parke’s` this way?
Suggest:
+ - Replace with: “Park's”
- Replace with: “Parker's”
- Replace with: “Parks's”
- - Replace with: “Park's”
@@ -6959,7 +6940,7 @@ Message: |
Suggest:
- Replace with: “Co”
- Replace with: “Cu”
- - Replace with: “CI”
+ - Replace with: “CA”
@@ -7107,7 +7088,7 @@ Message: |
Suggest:
- Replace with: “Be”
- Replace with: “Bi”
- - Replace with: “BO”
+ - Replace with: “BA”
@@ -7311,8 +7292,8 @@ Message: |
| ^~~~~~~~ Did you mean to spell `Hopalong` this way?
Suggest:
- Replace with: “Hopping”
+ - Replace with: “Halon”
- Replace with: “Haling”
- - Replace with: “Haloing”
diff --git a/harper-core/tests/text/tagged/Alice's Adventures in Wonderland.md b/harper-core/tests/text/tagged/Alice's Adventures in Wonderland.md
index 4ca5e1ebc..4b7dcf6b0 100644
--- a/harper-core/tests/text/tagged/Alice's Adventures in Wonderland.md
+++ b/harper-core/tests/text/tagged/Alice's Adventures in Wonderland.md
@@ -1161,7 +1161,7 @@
> RABBIT , ” engraved upon it . She went in without knocking , and hurried upstairs ,
# NSg/VB+ . . VP/J P NSg/ISg+ . ISg+ VPt NPr/J/R/P C/P Nᴹ/Vg/J . VB/C VP/J NSg/J .
> in great fear lest she should meet the real Mary Ann , and be turned out of the
-# NPr/J/R/P NSg/J+ N🅪Sg/VB+ JS ISg+ VXB NSg/VB/J D+ NSg/J+ NPr+ NPr/J+ . VB/C NSg/VLXB VP/J NSg/VB/J/R/P P D+
+# NPr/J/R/P NSg/J+ N🅪Sg/VB+ W? ISg+ VXB NSg/VB/J D+ NSg/J+ NPr+ NPr/J+ . VB/C NSg/VLXB VP/J NSg/VB/J/R/P P D+
> house before she had found the fan and gloves .
# NPr/VB+ C/P ISg+ VP NSg/VP D NSg/VB VB/C NPl/V3+ .
>
@@ -5699,7 +5699,7 @@
> must have meant some mischief , or else you’d have signed your name like an
# NSg/VXB NSg/VXB VP I/J/R/Dq+ NSg/VB+ . NPr/C NSg/J/C K NSg/VXB VP/J D$+ NSg/VB+ NSg/VB/J/C/P D/P
> honest man . ”
-# VB/JS NPr/VB+ . .
+# VB/J NPr/VB+ . .
>
#
> There was a general clapping of hands at this : it was the first really clever
diff --git a/harper-core/tests/text/tagged/The Great Gatsby.md b/harper-core/tests/text/tagged/The Great Gatsby.md
index d26208e63..cd32c8845 100644
--- a/harper-core/tests/text/tagged/The Great Gatsby.md
+++ b/harper-core/tests/text/tagged/The Great Gatsby.md
@@ -811,7 +811,7 @@
>
#
> “ Well , he wasn’t always a butler ; he used to be the silver polisher for some
-# . NSg/VB/J/R . NPr/ISg+ VPt R D/P NPr/VB . NPr/ISg+ VP/J P NSg/VLXB D Nᴹ/VB/J+ NSg/JC R/C/P I/J/R/Dq
+# . NSg/VB/J/R . NPr/ISg+ VPt R D/P NPr/VB . NPr/ISg+ VP/J P NSg/VLXB D Nᴹ/VB/J+ NSg R/C/P I/J/R/Dq
> people in New York that had a silver service for two hundred people . He had to
# NPl/VB+ NPr/J/R/P NSg/J NPr+ I/C/Ddem+ VP D/P Nᴹ/VB/J+ NSg/VB+ R/C/P NSg NSg NPl/VB+ . NPr/ISg+ VP P
> polish it from morning till night , until finally it began to affect his nose — — — ”
@@ -3653,7 +3653,7 @@
> Every one suspects himself of at least one of the cardinal virtues , and this is
# Dq NSg/I/J NPl/V3 ISg+ P NSg/P NSg/J/Dq NSg/I/J P D+ NSg/J+ NPl+ . VB/C I/Ddem+ VL3
> mine : I am one of the few honest people that I have ever known .
-# NSg/I/VB+ . ISg/#r+ NPr/VLB/J NSg/I/J P D+ NSg/I/Dq+ VB/JS+ NPl/VB+ I/C/Ddem+ ISg/#r+ NSg/VXB J/R VPp/J .
+# NSg/I/VB+ . ISg/#r+ NPr/VLB/J NSg/I/J P D+ NSg/I/Dq+ VB/J+ NPl/VB+ I/C/Ddem+ ISg/#r+ NSg/VXB J/R VPp/J .
>
#
> CHAPTER IV
@@ -11463,7 +11463,7 @@
> met another bad driver , didn’t I ? I mean it was careless of me to make such a
# VP I/D+ NSg/VB/J+ NSg+ . VXPt ISg/#r+ . ISg/#r+ NSg/VB/J NSg/ISg+ VLPt J P NPr/ISg+ P NSg/VB NSg/I D/P
> wrong guess . I thought you were rather an honest , straightforward person . I
-# NSg/VB/J/R NSg/VB+ . ISg/#r+ N🅪Sg/VP ISgPl+ VLPt NPr/R D/P VB/JS . J+ NSg/VB+ . ISg/#r+
+# NSg/VB/J/R NSg/VB+ . ISg/#r+ N🅪Sg/VP ISgPl+ VLPt NPr/R D/P VB/J . J+ NSg/VB+ . ISg/#r+
> thought it was your secret pride . ”
# N🅪Sg/VP NSg/ISg+ VLPt D$+ NSg/VB/J+ Nᴹ/VB+ . .
>
diff --git a/harper-desktop/src-tauri/Cargo.lock b/harper-desktop/src-tauri/Cargo.lock
index e04041427..936d4e687 100644
--- a/harper-desktop/src-tauri/Cargo.lock
+++ b/harper-desktop/src-tauri/Cargo.lock
@@ -3785,6 +3785,7 @@ dependencies = [
"harper-brill",
"harper-thesaurus",
"hashbrown 0.16.1",
+ "indexmap 2.14.2",
"is-macro",
"itertools 0.15.0",
"levenshtein_automata",