Offline, privacy-first grammar checker. Fast, open-source, Rust-powered
fix(#1876): merge dialects from curated, file, user, and workspace dicts (#1922)
Grant Lemons0d4feee7029b
diff --git a/harper-core/src/document.rs b/harper-core/src/document.rsindex 875e99893..e584ec385 100644--- a/harper-core/src/document.rs+++ b/harper-core/src/document.rs@@ -162,7 +162,9 @@ impl Document {for token in sent.iter_mut() {if let TokenKind::Word(meta) = &mut token.kind {let word_source = token.span.get_content(&self.source);- let mut found_meta = dictionary.get_word_metadata(word_source).cloned();+ let mut found_meta = dictionary+ .get_word_metadata(word_source)+ .map(|c| c.into_owned());if let Some(inner) = &mut found_meta {inner.pos_tag = token_tags[i].or_else(|| inner.infer_pos_tag());diff --git a/harper-core/src/expr/mergeable_words.rs b/harper-core/src/expr/mergeable_words.rsindex 02934cc4f..dffdc5f81 100644--- a/harper-core/src/expr/mergeable_words.rs+++ b/harper-core/src/expr/mergeable_words.rs@@ -51,7 +51,7 @@ impl MergeableWords {compound.remove(a_chars.len());let meta_closed = self.dict.get_word_metadata(&compound);- if (self.predicate)(meta_closed, meta_open) {+ if (self.predicate)(meta_closed.as_deref(), meta_open.as_deref()) {return Some(compound);}diff --git a/harper-core/src/linting/spell_check.rs b/harper-core/src/linting/spell_check.rsindex bd9d1d7fe..5f4a82dcb 100644--- a/harper-core/src/linting/spell_check.rs+++ b/harper-core/src/linting/spell_check.rs@@ -129,13 +129,16 @@ impl<T: Dictionary> Linter for SpellCheck<T> {#[cfg(test)]mod tests {use super::SpellCheck;- use crate::spell::FstDictionary;+ use crate::linting::Linter;+ use crate::spell::{Dictionary, FstDictionary, MergedDictionary, MutableDictionary};+ use crate::word_metadata::DialectFlags;use crate::{Dialect,linting::tests::{assert_lint_count, assert_suggestion_result, assert_top3_suggestion_result,},};+ use crate::{Document, WordMetadata};// Capitalization tests@@ -404,4 +407,47 @@ mod tests {"she's",);}++ #[test]+ fn issue_1876() {+ let user_dialect = Dialect::American;++ // Create a user dictionary with a word normally of another dialect in it.+ let mut user_dict = MutableDictionary::new();+ user_dict.append_word_str(+ "Calibre",+ WordMetadata {+ dialects: DialectFlags::from_dialect(user_dialect),+ ..Default::default()+ },+ );++ // Create a merged dictionary, using curated first.+ let mut merged_dict = MergedDictionary::new();+ merged_dict.add_dictionary(FstDictionary::curated());+ merged_dict.add_dictionary(std::sync::Arc::from(user_dict));+ assert!(merged_dict.contains_word_str("Calibre"));++ // No dialect issues should be found if the word from another dialect is in our user dictionary.+ assert_eq!(+ SpellCheck::new(merged_dict.clone(), user_dialect)+ .lint(&Document::new_markdown_default(+ "I like to use the software Calibre.",+ &merged_dict+ ))+ .len(),+ 0,+ "Calibre is not part of the user's dialect!"+ );++ assert_eq!(+ SpellCheck::new(merged_dict.clone(), user_dialect)+ .lint(&Document::new_markdown_default(+ "I like to use the spelling colour.",+ &merged_dict+ ))+ .len(),+ 1+ );+ }}diff --git a/harper-core/src/spell/dictionary.rs b/harper-core/src/spell/dictionary.rsindex 9d067c657..9f70a632b 100644--- a/harper-core/src/spell/dictionary.rs+++ b/harper-core/src/spell/dictionary.rs@@ -1,4 +1,5 @@use blanket::blanket;+use std::borrow::Cow;use super::FuzzyMatchResult;use super::WordId;@@ -33,11 +34,11 @@ pub trait Dictionary: Send + Sync {) -> Vec<FuzzyMatchResult<'_>>;fn get_correct_capitalization_of(&self, word: &[char]) -> Option<&'_ [char]>;/// Get the associated [`WordMetadata`] for any capitalization of a given word.- fn get_word_metadata(&self, word: &[char]) -> Option<&WordMetadata>;+ fn get_word_metadata(&self, word: &[char]) -> Option<Cow<'_, WordMetadata>>;/// Get the associated [`WordMetadata`] for any capitalization of a given word./// If the word isn't in the dictionary, the resulting metadata will be/// empty.- fn get_word_metadata_str(&self, word: &str) -> Option<&WordMetadata>;+ fn get_word_metadata_str(&self, word: &str) -> Option<Cow<'_, WordMetadata>>;/// Iterate over the words in the dictionary.fn words_iter(&self) -> Box<dyn Iterator<Item = &'_ [char]> + Send + '_>;diff --git a/harper-core/src/spell/fst_dictionary.rs b/harper-core/src/spell/fst_dictionary.rsindex 8f5eb010e..6d53b10a5 100644--- a/harper-core/src/spell/fst_dictionary.rs+++ b/harper-core/src/spell/fst_dictionary.rs@@ -2,6 +2,7 @@ use super::{MutableDictionary, WordId};use fst::{IntoStreamer, Map as FstMap, Streamer, map::StreamWithState};use lazy_static::lazy_static;use levenshtein_automata::{DFA, LevenshteinAutomatonBuilder};+use std::borrow::Cow;use std::{cell::RefCell, sync::Arc};use crate::{CharString, CharStringExt, WordMetadata};@@ -15,7 +16,7 @@ use super::FuzzyMatchResult;/// [`MutableDictionary`].pub struct FstDictionary {/// Underlying [`super::MutableDictionary`] used for everything except fuzzy finding- full_dict: Arc<MutableDictionary>,+ mutable_dict: Arc<MutableDictionary>,/// Used for fuzzy-finding the index of words or metadataword_map: FstMap<Vec<u8>>,/// Used for fuzzy-finding the index of words or metadata@@ -42,7 +43,7 @@ thread_local! {impl PartialEq for FstDictionary {fn eq(&self, other: &Self) -> bool {- self.full_dict == other.full_dict+ self.mutable_dict == other.mutable_dict}}@@ -67,14 +68,14 @@ impl FstDictionary {.expect("Insertion not in lexicographical order!");}- let mut full_dict = MutableDictionary::new();- full_dict.extend_words(words.iter().cloned());+ let mut mutable_dict = MutableDictionary::new();+ mutable_dict.extend_words(words.iter().cloned());let fst_bytes = builder.into_inner().unwrap();let word_map = FstMap::new(fst_bytes).expect("Unable to build FST map.");FstDictionary {- full_dict: Arc::new(full_dict),+ mutable_dict: Arc::new(mutable_dict),word_map,words,}@@ -113,19 +114,19 @@ fn stream_distances_vec(stream: &mut StreamWithState<&DFA>, dfa: &DFA) -> Vec<(uimpl Dictionary for FstDictionary {fn contains_word(&self, word: &[char]) -> bool {- self.full_dict.contains_word(word)+ self.mutable_dict.contains_word(word)}fn contains_word_str(&self, word: &str) -> bool {- self.full_dict.contains_word_str(word)+ self.mutable_dict.contains_word_str(word)}- fn get_word_metadata(&self, word: &[char]) -> Option<&WordMetadata> {- self.full_dict.get_word_metadata(word)+ fn get_word_metadata(&self, word: &[char]) -> Option<Cow<'_, WordMetadata>> {+ self.mutable_dict.get_word_metadata(word)}- fn get_word_metadata_str(&self, word: &str) -> Option<&WordMetadata> {- self.full_dict.get_word_metadata_str(word)+ fn get_word_metadata_str(&self, word: &str) -> Option<Cow<'_, WordMetadata>> {+ self.mutable_dict.get_word_metadata_str(word)}fn fuzzy_match(@@ -164,7 +165,7 @@ impl Dictionary for FstDictionary {merged.push(FuzzyMatchResult {word,edit_distance,- metadata,+ metadata: Cow::Borrowed(metadata),})}@@ -190,27 +191,27 @@ impl Dictionary for FstDictionary {}fn words_iter(&self) -> Box<dyn Iterator<Item = &'_ [char]> + Send + '_> {- self.full_dict.words_iter()+ self.mutable_dict.words_iter()}fn word_count(&self) -> usize {- self.full_dict.word_count()+ self.mutable_dict.word_count()}fn contains_exact_word(&self, word: &[char]) -> bool {- self.full_dict.contains_exact_word(word)+ self.mutable_dict.contains_exact_word(word)}fn contains_exact_word_str(&self, word: &str) -> bool {- self.full_dict.contains_exact_word_str(word)+ self.mutable_dict.contains_exact_word_str(word)}fn get_correct_capitalization_of(&self, word: &[char]) -> Option<&'_ [char]> {- self.full_dict.get_correct_capitalization_of(word)+ self.mutable_dict.get_correct_capitalization_of(word)}fn get_word_from_id(&self, id: &WordId) -> Option<&[char]> {- self.full_dict.get_word_from_id(id)+ self.mutable_dict.get_word_from_id(id)}}@@ -244,7 +245,7 @@ mod tests {}#[test]- fn fst_map_contains_all_in_full_dict() {+ fn fst_map_contains_all_in_mutable_dict() {let dict = FstDictionary::curated();for word in dict.words_iter() {diff --git a/harper-core/src/spell/merged_dictionary.rs b/harper-core/src/spell/merged_dictionary.rsindex 7c8863547..f080efb8c 100644--- a/harper-core/src/spell/merged_dictionary.rs+++ b/harper-core/src/spell/merged_dictionary.rs@@ -1,3 +1,4 @@+use std::borrow::Cow;use std::hash::{BuildHasher, Hasher};use std::sync::Arc;@@ -93,14 +94,11 @@ impl Dictionary for MergedDictionary {false}- fn get_word_metadata(&self, word: &[char]) -> Option<&WordMetadata> {- for child in &self.children {- if let Some(found_item) = child.get_word_metadata(word) {- return Some(found_item);- }- }-- None+ fn get_word_metadata(&self, word: &[char]) -> Option<Cow<'_, WordMetadata>> {+ self.children+ .iter()+ .filter_map(|d| d.get_word_metadata(word))+ .reduce(|acc, md| Cow::Owned(acc.or(&md)))}fn words_iter(&self) -> Box<dyn Iterator<Item = &'_ [char]> + Send + '_> {@@ -117,7 +115,7 @@ impl Dictionary for MergedDictionary {self.contains_word(&chars)}- fn get_word_metadata_str(&self, word: &str) -> Option<&WordMetadata> {+ fn get_word_metadata_str(&self, word: &str) -> Option<Cow<'_, WordMetadata>> {let chars: CharString = word.chars().collect();self.get_word_metadata(&chars)}diff --git a/harper-core/src/spell/mod.rs b/harper-core/src/spell/mod.rsindex 45c4f64e1..c02f8812d 100644--- a/harper-core/src/spell/mod.rs+++ b/harper-core/src/spell/mod.rs@@ -21,7 +21,7 @@ mod word_map;pub struct FuzzyMatchResult<'a> {pub word: &'a [char],pub edit_distance: u8,- pub metadata: &'a WordMetadata,+ pub metadata: std::borrow::Cow<'a, WordMetadata>,}impl PartialOrd for FuzzyMatchResult<'_> {diff --git a/harper-core/src/spell/mutable_dictionary.rs b/harper-core/src/spell/mutable_dictionary.rsindex f6799fb17..7f179073b 100644--- a/harper-core/src/spell/mutable_dictionary.rs+++ b/harper-core/src/spell/mutable_dictionary.rs@@ -6,6 +6,7 @@ use super::{use crate::edit_distance::edit_distance_min_alloc;use itertools::Itertools;use lazy_static::lazy_static;+use std::borrow::Cow;use std::sync::Arc;use crate::{CharString, CharStringExt, WordMetadata};@@ -107,8 +108,10 @@ impl Default for MutableDictionary {}impl Dictionary for MutableDictionary {- fn get_word_metadata(&self, word: &[char]) -> Option<&WordMetadata> {- self.word_map.get_with_chars(word).map(|v| &v.metadata)+ fn get_word_metadata(&self, word: &[char]) -> Option<Cow<'_, WordMetadata>> {+ self.word_map+ .get_with_chars(word)+ .map(|v| Cow::Borrowed(&v.metadata))}fn contains_word(&self, word: &[char]) -> bool {@@ -120,7 +123,7 @@ impl Dictionary for MutableDictionary {self.contains_word(&chars)}- fn get_word_metadata_str(&self, word: &str) -> Option<&WordMetadata> {+ fn get_word_metadata_str(&self, word: &str) -> Option<Cow<'_, WordMetadata>> {let chars: CharString = word.chars().collect();self.get_word_metadata(&chars)}diff --git a/harper-core/src/title_case.rs b/harper-core/src/title_case.rsindex ef81c5208..453825abc 100644--- a/harper-core/src/title_case.rs+++ b/harper-core/src/title_case.rs@@ -91,7 +91,7 @@ fn should_capitalize_token(tok: &Token, source: &[char], dict: &impl Dictionary)let mut metadata = Cow::Borrowed(metadata);if let Some(metadata_lower) = dict.get_word_metadata(&chars_lower) {- metadata = Cow::Owned(metadata.clone().or(metadata_lower));+ metadata = Cow::Owned(metadata.clone().or(&metadata_lower));}let is_short_preposition = metadata.preposition && tok.span.len() <= 4;diff --git a/harper-core/src/word_metadata.rs b/harper-core/src/word_metadata.rsindex 0d8457d82..b6ca57630 100644--- a/harper-core/src/word_metadata.rs+++ b/harper-core/src/word_metadata.rs@@ -1145,7 +1145,7 @@ pub mod tests {FstDictionary::curated().get_word_metadata_str(word).unwrap_or_else(|| panic!("Word '{word}' not found in dictionary"))- .clone()+ .into_owned()}mod dialect {diff --git a/harper-ls/src/backend.rs b/harper-ls/src/backend.rsindex 470a7449f..03e64cca7 100644--- a/harper-ls/src/backend.rs+++ b/harper-ls/src/backend.rs@@ -79,7 +79,7 @@ impl Backend {.await.context("Unable to get the file path.")?;- load_dict(path)+ load_dict(path, self.config.read().await.dialect).await.map_err(|err| info!("{err}")).or(Ok(MutableDictionary::new()))@@ -143,7 +143,7 @@ impl Backend {async fn load_user_dictionary(&self) -> MutableDictionary {let config = self.config.read().await;- load_dict(&config.user_dict_path)+ load_dict(&config.user_dict_path, self.config.read().await.dialect).await.map_err(|err| info!("{err}")).unwrap_or(MutableDictionary::new())@@ -159,10 +159,13 @@ impl Backend {async fn load_workspace_dictionary(&self) -> MutableDictionary {let config = self.config.read().await;- load_dict(&config.workspace_dict_path)- .await- .map_err(|err| info!("{err}"))- .unwrap_or(MutableDictionary::new())+ load_dict(+ &config.workspace_dict_path,+ self.config.read().await.dialect,+ )+ .await+ .map_err(|err| info!("{err}"))+ .unwrap_or(MutableDictionary::new())}async fn save_workspace_dictionary(&self, dict: impl Dictionary) -> Result<()> {diff --git a/harper-ls/src/dictionary_io.rs b/harper-ls/src/dictionary_io.rsindex 1a284e235..de0a12f3a 100644--- a/harper-ls/src/dictionary_io.rs+++ b/harper-ls/src/dictionary_io.rs@@ -1,8 +1,9 @@+use harper_core::word_metadata::DialectFlags;use itertools::Itertools;use std::path::Path;-use harper_core::WordMetadata;use harper_core::spell::{Dictionary, MutableDictionary};+use harper_core::{Dialect, WordMetadata};use tokio::fs::{self, File};use tokio::io::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt, BufReader, BufWriter, Result};@@ -38,26 +39,34 @@ async fn write_word_list(dict: impl Dictionary, mut w: impl AsyncWrite + Unpin)}/// Load a dictionary from a file on disk.-pub async fn load_dict(path: impl AsRef<Path>) -> Result<MutableDictionary> {+pub async fn load_dict(path: impl AsRef<Path>, dialect: Dialect) -> Result<MutableDictionary> {let file = File::open(path.as_ref()).await?;let read = BufReader::new(file);- dict_from_word_list(read).await+ dict_from_word_list(read, dialect).await}/// Load a dictionary from a list of words./// It could definitely be optimized to use less memory./// Right now it isn't an issue.-async fn dict_from_word_list(mut r: impl AsyncRead + Unpin) -> Result<MutableDictionary> {+async fn dict_from_word_list(+ mut r: impl AsyncRead + Unpin,+ dialect: Dialect,+) -> Result<MutableDictionary> {let mut str = String::new();r.read_to_string(&mut str).await?;let mut dict = MutableDictionary::new();- dict.extend_words(- str.lines()- .map(|l| (l.chars().collect::<Vec<char>>(), WordMetadata::default())),- );+ dict.extend_words(str.lines().map(|l| {+ (+ l.chars().collect::<Vec<char>>(),+ WordMetadata {+ dialects: DialectFlags::from_dialect(dialect),+ ..Default::default()+ },+ )+ }));Ok(dict)}