Skip to content

Offline, privacy-first grammar checker. Fast, open-source, Rust-powered

fix(#1876): merge dialects from curated, file, user, and workspace dicts (#1922)

Grant Lemons0d4feee7029b
diff --git a/harper-core/src/document.rs b/harper-core/src/document.rs
index 875e99893..e584ec385 100644
--- a/harper-core/src/document.rs
+++ b/harper-core/src/document.rs
@@ -162,7 +162,9 @@ impl Document {
for token in sent.iter_mut() {
if let TokenKind::Word(meta) = &mut token.kind {
let word_source = token.span.get_content(&self.source);
- let mut found_meta = dictionary.get_word_metadata(word_source).cloned();
+ let mut found_meta = dictionary
+ .get_word_metadata(word_source)
+ .map(|c| c.into_owned());
if let Some(inner) = &mut found_meta {
inner.pos_tag = token_tags[i].or_else(|| inner.infer_pos_tag());
diff --git a/harper-core/src/expr/mergeable_words.rs b/harper-core/src/expr/mergeable_words.rs
index 02934cc4f..dffdc5f81 100644
--- a/harper-core/src/expr/mergeable_words.rs
+++ b/harper-core/src/expr/mergeable_words.rs
@@ -51,7 +51,7 @@ impl MergeableWords {
compound.remove(a_chars.len());
let meta_closed = self.dict.get_word_metadata(&compound);
- if (self.predicate)(meta_closed, meta_open) {
+ if (self.predicate)(meta_closed.as_deref(), meta_open.as_deref()) {
return Some(compound);
}
diff --git a/harper-core/src/linting/spell_check.rs b/harper-core/src/linting/spell_check.rs
index bd9d1d7fe..5f4a82dcb 100644
--- a/harper-core/src/linting/spell_check.rs
+++ b/harper-core/src/linting/spell_check.rs
@@ -129,13 +129,16 @@ impl<T: Dictionary> Linter for SpellCheck<T> {
#[cfg(test)]
mod tests {
use super::SpellCheck;
- use crate::spell::FstDictionary;
+ use crate::linting::Linter;
+ use crate::spell::{Dictionary, FstDictionary, MergedDictionary, MutableDictionary};
+ use crate::word_metadata::DialectFlags;
use crate::{
Dialect,
linting::tests::{
assert_lint_count, assert_suggestion_result, assert_top3_suggestion_result,
},
};
+ use crate::{Document, WordMetadata};
// Capitalization tests
@@ -404,4 +407,47 @@ mod tests {
"she's",
);
}
+
+ #[test]
+ fn issue_1876() {
+ let user_dialect = Dialect::American;
+
+ // Create a user dictionary with a word normally of another dialect in it.
+ let mut user_dict = MutableDictionary::new();
+ user_dict.append_word_str(
+ "Calibre",
+ WordMetadata {
+ dialects: DialectFlags::from_dialect(user_dialect),
+ ..Default::default()
+ },
+ );
+
+ // Create a merged dictionary, using curated first.
+ let mut merged_dict = MergedDictionary::new();
+ merged_dict.add_dictionary(FstDictionary::curated());
+ merged_dict.add_dictionary(std::sync::Arc::from(user_dict));
+ assert!(merged_dict.contains_word_str("Calibre"));
+
+ // No dialect issues should be found if the word from another dialect is in our user dictionary.
+ assert_eq!(
+ SpellCheck::new(merged_dict.clone(), user_dialect)
+ .lint(&Document::new_markdown_default(
+ "I like to use the software Calibre.",
+ &merged_dict
+ ))
+ .len(),
+ 0,
+ "Calibre is not part of the user's dialect!"
+ );
+
+ assert_eq!(
+ SpellCheck::new(merged_dict.clone(), user_dialect)
+ .lint(&Document::new_markdown_default(
+ "I like to use the spelling colour.",
+ &merged_dict
+ ))
+ .len(),
+ 1
+ );
+ }
}
diff --git a/harper-core/src/spell/dictionary.rs b/harper-core/src/spell/dictionary.rs
index 9d067c657..9f70a632b 100644
--- a/harper-core/src/spell/dictionary.rs
+++ b/harper-core/src/spell/dictionary.rs
@@ -1,4 +1,5 @@
use blanket::blanket;
+use std::borrow::Cow;
use super::FuzzyMatchResult;
use super::WordId;
@@ -33,11 +34,11 @@ pub trait Dictionary: Send + Sync {
) -> Vec<FuzzyMatchResult<'_>>;
fn get_correct_capitalization_of(&self, word: &[char]) -> Option<&'_ [char]>;
/// Get the associated [`WordMetadata`] for any capitalization of a given word.
- fn get_word_metadata(&self, word: &[char]) -> Option<&WordMetadata>;
+ fn get_word_metadata(&self, word: &[char]) -> Option<Cow<'_, WordMetadata>>;
/// Get the associated [`WordMetadata`] for any capitalization of a given word.
/// If the word isn't in the dictionary, the resulting metadata will be
/// empty.
- fn get_word_metadata_str(&self, word: &str) -> Option<&WordMetadata>;
+ fn get_word_metadata_str(&self, word: &str) -> Option<Cow<'_, WordMetadata>>;
/// Iterate over the words in the dictionary.
fn words_iter(&self) -> Box<dyn Iterator<Item = &'_ [char]> + Send + '_>;
diff --git a/harper-core/src/spell/fst_dictionary.rs b/harper-core/src/spell/fst_dictionary.rs
index 8f5eb010e..6d53b10a5 100644
--- a/harper-core/src/spell/fst_dictionary.rs
+++ b/harper-core/src/spell/fst_dictionary.rs
@@ -2,6 +2,7 @@ use super::{MutableDictionary, WordId};
use fst::{IntoStreamer, Map as FstMap, Streamer, map::StreamWithState};
use lazy_static::lazy_static;
use levenshtein_automata::{DFA, LevenshteinAutomatonBuilder};
+use std::borrow::Cow;
use std::{cell::RefCell, sync::Arc};
use crate::{CharString, CharStringExt, WordMetadata};
@@ -15,7 +16,7 @@ use super::FuzzyMatchResult;
/// [`MutableDictionary`].
pub struct FstDictionary {
/// Underlying [`super::MutableDictionary`] used for everything except fuzzy finding
- full_dict: Arc<MutableDictionary>,
+ mutable_dict: Arc<MutableDictionary>,
/// Used for fuzzy-finding the index of words or metadata
word_map: FstMap<Vec<u8>>,
/// Used for fuzzy-finding the index of words or metadata
@@ -42,7 +43,7 @@ thread_local! {
impl PartialEq for FstDictionary {
fn eq(&self, other: &Self) -> bool {
- self.full_dict == other.full_dict
+ self.mutable_dict == other.mutable_dict
}
}
@@ -67,14 +68,14 @@ impl FstDictionary {
.expect("Insertion not in lexicographical order!");
}
- let mut full_dict = MutableDictionary::new();
- full_dict.extend_words(words.iter().cloned());
+ let mut mutable_dict = MutableDictionary::new();
+ mutable_dict.extend_words(words.iter().cloned());
let fst_bytes = builder.into_inner().unwrap();
let word_map = FstMap::new(fst_bytes).expect("Unable to build FST map.");
FstDictionary {
- full_dict: Arc::new(full_dict),
+ mutable_dict: Arc::new(mutable_dict),
word_map,
words,
}
@@ -113,19 +114,19 @@ fn stream_distances_vec(stream: &mut StreamWithState<&DFA>, dfa: &DFA) -> Vec<(u
impl Dictionary for FstDictionary {
fn contains_word(&self, word: &[char]) -> bool {
- self.full_dict.contains_word(word)
+ self.mutable_dict.contains_word(word)
}
fn contains_word_str(&self, word: &str) -> bool {
- self.full_dict.contains_word_str(word)
+ self.mutable_dict.contains_word_str(word)
}
- fn get_word_metadata(&self, word: &[char]) -> Option<&WordMetadata> {
- self.full_dict.get_word_metadata(word)
+ fn get_word_metadata(&self, word: &[char]) -> Option<Cow<'_, WordMetadata>> {
+ self.mutable_dict.get_word_metadata(word)
}
- fn get_word_metadata_str(&self, word: &str) -> Option<&WordMetadata> {
- self.full_dict.get_word_metadata_str(word)
+ fn get_word_metadata_str(&self, word: &str) -> Option<Cow<'_, WordMetadata>> {
+ self.mutable_dict.get_word_metadata_str(word)
}
fn fuzzy_match(
@@ -164,7 +165,7 @@ impl Dictionary for FstDictionary {
merged.push(FuzzyMatchResult {
word,
edit_distance,
- metadata,
+ metadata: Cow::Borrowed(metadata),
})
}
@@ -190,27 +191,27 @@ impl Dictionary for FstDictionary {
}
fn words_iter(&self) -> Box<dyn Iterator<Item = &'_ [char]> + Send + '_> {
- self.full_dict.words_iter()
+ self.mutable_dict.words_iter()
}
fn word_count(&self) -> usize {
- self.full_dict.word_count()
+ self.mutable_dict.word_count()
}
fn contains_exact_word(&self, word: &[char]) -> bool {
- self.full_dict.contains_exact_word(word)
+ self.mutable_dict.contains_exact_word(word)
}
fn contains_exact_word_str(&self, word: &str) -> bool {
- self.full_dict.contains_exact_word_str(word)
+ self.mutable_dict.contains_exact_word_str(word)
}
fn get_correct_capitalization_of(&self, word: &[char]) -> Option<&'_ [char]> {
- self.full_dict.get_correct_capitalization_of(word)
+ self.mutable_dict.get_correct_capitalization_of(word)
}
fn get_word_from_id(&self, id: &WordId) -> Option<&[char]> {
- self.full_dict.get_word_from_id(id)
+ self.mutable_dict.get_word_from_id(id)
}
}
@@ -244,7 +245,7 @@ mod tests {
}
#[test]
- fn fst_map_contains_all_in_full_dict() {
+ fn fst_map_contains_all_in_mutable_dict() {
let dict = FstDictionary::curated();
for word in dict.words_iter() {
diff --git a/harper-core/src/spell/merged_dictionary.rs b/harper-core/src/spell/merged_dictionary.rs
index 7c8863547..f080efb8c 100644
--- a/harper-core/src/spell/merged_dictionary.rs
+++ b/harper-core/src/spell/merged_dictionary.rs
@@ -1,3 +1,4 @@
+use std::borrow::Cow;
use std::hash::{BuildHasher, Hasher};
use std::sync::Arc;
@@ -93,14 +94,11 @@ impl Dictionary for MergedDictionary {
false
}
- fn get_word_metadata(&self, word: &[char]) -> Option<&WordMetadata> {
- for child in &self.children {
- if let Some(found_item) = child.get_word_metadata(word) {
- return Some(found_item);
- }
- }
-
- None
+ fn get_word_metadata(&self, word: &[char]) -> Option<Cow<'_, WordMetadata>> {
+ self.children
+ .iter()
+ .filter_map(|d| d.get_word_metadata(word))
+ .reduce(|acc, md| Cow::Owned(acc.or(&md)))
}
fn words_iter(&self) -> Box<dyn Iterator<Item = &'_ [char]> + Send + '_> {
@@ -117,7 +115,7 @@ impl Dictionary for MergedDictionary {
self.contains_word(&chars)
}
- fn get_word_metadata_str(&self, word: &str) -> Option<&WordMetadata> {
+ fn get_word_metadata_str(&self, word: &str) -> Option<Cow<'_, WordMetadata>> {
let chars: CharString = word.chars().collect();
self.get_word_metadata(&chars)
}
diff --git a/harper-core/src/spell/mod.rs b/harper-core/src/spell/mod.rs
index 45c4f64e1..c02f8812d 100644
--- a/harper-core/src/spell/mod.rs
+++ b/harper-core/src/spell/mod.rs
@@ -21,7 +21,7 @@ mod word_map;
pub struct FuzzyMatchResult<'a> {
pub word: &'a [char],
pub edit_distance: u8,
- pub metadata: &'a WordMetadata,
+ pub metadata: std::borrow::Cow<'a, WordMetadata>,
}
impl PartialOrd for FuzzyMatchResult<'_> {
diff --git a/harper-core/src/spell/mutable_dictionary.rs b/harper-core/src/spell/mutable_dictionary.rs
index f6799fb17..7f179073b 100644
--- a/harper-core/src/spell/mutable_dictionary.rs
+++ b/harper-core/src/spell/mutable_dictionary.rs
@@ -6,6 +6,7 @@ use super::{
use crate::edit_distance::edit_distance_min_alloc;
use itertools::Itertools;
use lazy_static::lazy_static;
+use std::borrow::Cow;
use std::sync::Arc;
use crate::{CharString, CharStringExt, WordMetadata};
@@ -107,8 +108,10 @@ impl Default for MutableDictionary {
}
impl Dictionary for MutableDictionary {
- fn get_word_metadata(&self, word: &[char]) -> Option<&WordMetadata> {
- self.word_map.get_with_chars(word).map(|v| &v.metadata)
+ fn get_word_metadata(&self, word: &[char]) -> Option<Cow<'_, WordMetadata>> {
+ self.word_map
+ .get_with_chars(word)
+ .map(|v| Cow::Borrowed(&v.metadata))
}
fn contains_word(&self, word: &[char]) -> bool {
@@ -120,7 +123,7 @@ impl Dictionary for MutableDictionary {
self.contains_word(&chars)
}
- fn get_word_metadata_str(&self, word: &str) -> Option<&WordMetadata> {
+ fn get_word_metadata_str(&self, word: &str) -> Option<Cow<'_, WordMetadata>> {
let chars: CharString = word.chars().collect();
self.get_word_metadata(&chars)
}
diff --git a/harper-core/src/title_case.rs b/harper-core/src/title_case.rs
index ef81c5208..453825abc 100644
--- a/harper-core/src/title_case.rs
+++ b/harper-core/src/title_case.rs
@@ -91,7 +91,7 @@ fn should_capitalize_token(tok: &Token, source: &[char], dict: &impl Dictionary)
let mut metadata = Cow::Borrowed(metadata);
if let Some(metadata_lower) = dict.get_word_metadata(&chars_lower) {
- metadata = Cow::Owned(metadata.clone().or(metadata_lower));
+ metadata = Cow::Owned(metadata.clone().or(&metadata_lower));
}
let is_short_preposition = metadata.preposition && tok.span.len() <= 4;
diff --git a/harper-core/src/word_metadata.rs b/harper-core/src/word_metadata.rs
index 0d8457d82..b6ca57630 100644
--- a/harper-core/src/word_metadata.rs
+++ b/harper-core/src/word_metadata.rs
@@ -1145,7 +1145,7 @@ pub mod tests {
FstDictionary::curated()
.get_word_metadata_str(word)
.unwrap_or_else(|| panic!("Word '{word}' not found in dictionary"))
- .clone()
+ .into_owned()
}
mod dialect {
diff --git a/harper-ls/src/backend.rs b/harper-ls/src/backend.rs
index 470a7449f..03e64cca7 100644
--- a/harper-ls/src/backend.rs
+++ b/harper-ls/src/backend.rs
@@ -79,7 +79,7 @@ impl Backend {
.await
.context("Unable to get the file path.")?;
- load_dict(path)
+ load_dict(path, self.config.read().await.dialect)
.await
.map_err(|err| info!("{err}"))
.or(Ok(MutableDictionary::new()))
@@ -143,7 +143,7 @@ impl Backend {
async fn load_user_dictionary(&self) -> MutableDictionary {
let config = self.config.read().await;
- load_dict(&config.user_dict_path)
+ load_dict(&config.user_dict_path, self.config.read().await.dialect)
.await
.map_err(|err| info!("{err}"))
.unwrap_or(MutableDictionary::new())
@@ -159,10 +159,13 @@ impl Backend {
async fn load_workspace_dictionary(&self) -> MutableDictionary {
let config = self.config.read().await;
- load_dict(&config.workspace_dict_path)
- .await
- .map_err(|err| info!("{err}"))
- .unwrap_or(MutableDictionary::new())
+ load_dict(
+ &config.workspace_dict_path,
+ self.config.read().await.dialect,
+ )
+ .await
+ .map_err(|err| info!("{err}"))
+ .unwrap_or(MutableDictionary::new())
}
async fn save_workspace_dictionary(&self, dict: impl Dictionary) -> Result<()> {
diff --git a/harper-ls/src/dictionary_io.rs b/harper-ls/src/dictionary_io.rs
index 1a284e235..de0a12f3a 100644
--- a/harper-ls/src/dictionary_io.rs
+++ b/harper-ls/src/dictionary_io.rs
@@ -1,8 +1,9 @@
+use harper_core::word_metadata::DialectFlags;
use itertools::Itertools;
use std::path::Path;
-use harper_core::WordMetadata;
use harper_core::spell::{Dictionary, MutableDictionary};
+use harper_core::{Dialect, WordMetadata};
use tokio::fs::{self, File};
use tokio::io::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt, BufReader, BufWriter, Result};
@@ -38,26 +39,34 @@ async fn write_word_list(dict: impl Dictionary, mut w: impl AsyncWrite + Unpin)
}
/// Load a dictionary from a file on disk.
-pub async fn load_dict(path: impl AsRef<Path>) -> Result<MutableDictionary> {
+pub async fn load_dict(path: impl AsRef<Path>, dialect: Dialect) -> Result<MutableDictionary> {
let file = File::open(path.as_ref()).await?;
let read = BufReader::new(file);
- dict_from_word_list(read).await
+ dict_from_word_list(read, dialect).await
}
/// Load a dictionary from a list of words.
/// It could definitely be optimized to use less memory.
/// Right now it isn't an issue.
-async fn dict_from_word_list(mut r: impl AsyncRead + Unpin) -> Result<MutableDictionary> {
+async fn dict_from_word_list(
+ mut r: impl AsyncRead + Unpin,
+ dialect: Dialect,
+) -> Result<MutableDictionary> {
let mut str = String::new();
r.read_to_string(&mut str).await?;
let mut dict = MutableDictionary::new();
- dict.extend_words(
- str.lines()
- .map(|l| (l.chars().collect::<Vec<char>>(), WordMetadata::default())),
- );
+ dict.extend_words(str.lines().map(|l| {
+ (
+ l.chars().collect::<Vec<char>>(),
+ WordMetadata {
+ dialects: DialectFlags::from_dialect(dialect),
+ ..Default::default()
+ },
+ )
+ }));
Ok(dict)
}