Skip to content

Offline, privacy-first grammar checker. Fast, open-source, Rust-powered

refactor(core): made `MutableDictionary` buildable from Rune files

Elijah Potter6f54df6f69c5
diff --git a/harper-core/src/ignored_lints/mod.rs b/harper-core/src/ignored_lints/mod.rs
index 834e19a82..84dc8337a 100644
--- a/harper-core/src/ignored_lints/mod.rs
+++ b/harper-core/src/ignored_lints/mod.rs
@@ -9,6 +9,9 @@ use serde::{Deserialize, Serialize};
use crate::{Document, linting::Lint};
/// A structure that keeps track of lints that have been ignored by users.
+///
+/// To use this structure, apply [`Self::remove_ignored`] on the output of a
+/// [`Linter`](crate::linting::Linter).
#[derive(Debug, Default, Serialize, Deserialize)]
pub struct IgnoredLints {
context_hashes: HashSet<u64>,
diff --git a/harper-core/src/spell/mutable_dictionary.rs b/harper-core/src/spell/mutable_dictionary.rs
index 19323a66f..bbdcf514b 100644
--- a/harper-core/src/spell/mutable_dictionary.rs
+++ b/harper-core/src/spell/mutable_dictionary.rs
@@ -1,6 +1,6 @@
use super::{
FstDictionary, WordId,
- rune::{parse_default_attribute_list, parse_default_word_list},
+ rune::{self, AttributeList, parse_word_list},
word_map::{WordMap, WordMapEntry},
};
use crate::edit_distance::edit_distance_min_alloc;
@@ -30,15 +30,13 @@ pub struct MutableDictionary {
/// The uncached function that is used to produce the original copy of the
/// curated dictionary.
fn uncached_inner_new() -> Arc<MutableDictionary> {
- let word_list = parse_default_word_list().unwrap();
- let attr_list = parse_default_attribute_list();
-
- // There will be at _least_ this number of words
- let mut word_map = WordMap::default();
-
- attr_list.expand_marked_words(word_list, &mut word_map);
-
- Arc::new(MutableDictionary { word_map })
+ Arc::new(
+ MutableDictionary::from_rune_files(
+ include_str!("../../dictionary.dict"),
+ include_str!("../../affixes.json"),
+ )
+ .expect("Curated dictionary should be valid."),
+ )
}
lazy_static! {
@@ -52,6 +50,18 @@ impl MutableDictionary {
}
}
+ pub fn from_rune_files(word_list: &str, attr_list: &str) -> Result<Self, rune::Error> {
+ let word_list = parse_word_list(word_list)?;
+ let attr_list = AttributeList::parse(attr_list)?;
+
+ // There will be at _least_ this number of words
+ let mut word_map = WordMap::default();
+
+ attr_list.expand_marked_words(word_list, &mut word_map);
+
+ Ok(Self { word_map })
+ }
+
/// Create a dictionary from the curated dictionary included
/// in the Harper binary.
/// Consider using [`super::FstDictionary::curated()`] instead, as it is more performant for spellchecking.
@@ -239,6 +249,7 @@ impl From<MutableDictionary> for FstDictionary {
#[cfg(test)]
mod tests {
+ use hashbrown::HashSet;
use itertools::Itertools;
use crate::{Dictionary, MutableDictionary};
@@ -313,4 +324,67 @@ mod tests {
assert!(!dict.get_word_metadata_str("there").unwrap().is_nominal());
assert!(!dict.get_word_metadata_str("there").unwrap().is_pronoun());
}
+
+ #[test]
+ fn expanded_contains_giants() {
+ assert!(MutableDictionary::curated().contains_word_str("giants"));
+ }
+
+ #[test]
+ fn expanded_contains_deallocate() {
+ assert!(MutableDictionary::curated().contains_word_str("deallocate"));
+ }
+
+ #[test]
+ fn curated_contains_repo() {
+ let dict = MutableDictionary::curated();
+
+ assert!(dict.contains_word_str("repo"));
+ assert!(dict.contains_word_str("repos"));
+ assert!(dict.contains_word_str("repo's"));
+ }
+
+ #[test]
+ fn curated_contains_possessive_abandonment() {
+ assert!(
+ MutableDictionary::curated()
+ .get_word_metadata_str("abandonment's")
+ .unwrap()
+ .is_possessive_noun()
+ )
+ }
+
+ #[test]
+ fn has_is_not_a_nominal() {
+ let dict = MutableDictionary::curated();
+
+ let has = dict.get_word_metadata_str("has");
+ assert!(has.is_some());
+
+ assert!(!has.unwrap().is_nominal())
+ }
+
+ #[test]
+ fn is_is_linking_verb() {
+ let dict = MutableDictionary::curated();
+
+ let is = dict.get_word_metadata_str("is");
+
+ assert!(is.is_some());
+ assert!(is.unwrap().is_linking_verb());
+ }
+
+ #[test]
+ fn are_merged_attrs_same_as_spread_attrs() {
+ let curated_attr_list = include_str!("../../affixes.json");
+
+ let merged = MutableDictionary::from_rune_files("1\nblork/DGS", curated_attr_list).unwrap();
+ let spread =
+ MutableDictionary::from_rune_files("2\nblork/DG\nblork/S", curated_attr_list).unwrap();
+
+ assert_eq!(
+ merged.word_map.into_iter().collect::<HashSet<_>>(),
+ spread.word_map.into_iter().collect::<HashSet<_>>()
+ );
+ }
}
diff --git a/harper-core/src/spell/rune/attribute_list.rs b/harper-core/src/spell/rune/attribute_list.rs
index 9cf8b2d4b..14d199ed9 100644
--- a/harper-core/src/spell/rune/attribute_list.rs
+++ b/harper-core/src/spell/rune/attribute_list.rs
@@ -16,7 +16,7 @@ pub struct AttributeList {
}
impl AttributeList {
- pub fn into_human_readable(self) -> HumanReadableAttributeList {
+ fn into_human_readable(self) -> HumanReadableAttributeList {
HumanReadableAttributeList {
affixes: self
.affixes
@@ -26,6 +26,13 @@ impl AttributeList {
}
}
+ pub fn parse(source: &str) -> Result<Self, Error> {
+ let human_readable: HumanReadableAttributeList =
+ serde_json::from_str(source).map_err(|_| Error::MalformedJSON)?;
+
+ human_readable.into_normal()
+ }
+
/// Expand [`MarkedWord`] into a list of full words, including itself.
///
/// Will append to the given `dest`;
diff --git a/harper-core/src/spell/rune/error.rs b/harper-core/src/spell/rune/error.rs
index 8f1e113d2..41233d86a 100644
--- a/harper-core/src/spell/rune/error.rs
+++ b/harper-core/src/spell/rune/error.rs
@@ -12,6 +12,8 @@ pub enum Error {
ExpectedUnsignedInteger,
#[error("Could not parse because we encountered the end of the line.")]
UnexpectedEndOfLine,
+ #[error("Received malformed JSON")]
+ MalformedJSON,
#[error("An error occured with a condition: {0}")]
Matcher(#[from] matcher::Error),
}
diff --git a/harper-core/src/spell/rune/mod.rs b/harper-core/src/spell/rune/mod.rs
index d8e332511..0a4badbc2 100644
--- a/harper-core/src/spell/rune/mod.rs
+++ b/harper-core/src/spell/rune/mod.rs
@@ -6,25 +6,9 @@ mod matcher;
pub mod word_list;
pub use attribute_list::AttributeList;
-use attribute_list::HumanReadableAttributeList;
pub use error::Error;
-pub use self::word_list::MarkedWord;
-use self::word_list::parse_word_list;
-
-pub fn parse_default_word_list() -> Result<Vec<MarkedWord>, Error> {
- parse_word_list(include_str!("../../../dictionary.dict"))
-}
-
-pub fn parse_default_attribute_list() -> AttributeList {
- let human_readable: HumanReadableAttributeList =
- serde_json::from_str(include_str!("../../../affixes.json"))
- .expect("The built-in affix list should always be valid.");
-
- human_readable
- .into_normal()
- .expect("All expressions in the built-in attribute list should be valid.")
-}
+pub use self::word_list::parse_word_list;
#[cfg(test)]
mod tests {
@@ -33,10 +17,9 @@ mod tests {
use serde_json::json;
use super::super::word_map::WordMap;
- use super::HumanReadableAttributeList;
use super::word_list::parse_word_list;
- use super::{parse_default_attribute_list, parse_default_word_list};
use crate::CharStringExt;
+ use crate::spell::rune::AttributeList;
pub const TEST_WORD_LIST: &str = "4\nhello\ntry/B\nwork/AB\nblank/";
@@ -93,12 +76,12 @@ mod tests {
fn assert_expansion_results(test_word_list: &str, expected: Vec<&str>) {
let words = parse_word_list(test_word_list).unwrap();
- let attributes: HumanReadableAttributeList =
- serde_json::from_value(TEST_AFFIX_JSON.clone()).unwrap();
- let attributes = attributes.into_normal().unwrap();
+ let attributes = AttributeList::parse(&TEST_AFFIX_JSON.to_string()).unwrap();
let mut expanded = WordMap::default();
+
attributes.expand_marked_words(words, &mut expanded);
+
let expanded: HashSet<String> = expanded
.into_iter()
.map(|v| v.canonical_spelling.into_iter().collect())
@@ -139,10 +122,7 @@ mod tests {
#[test]
fn correctly_expands_test_files_with_comments() {
let words = parse_word_list(TEST_WORD_LIST_WITH_COMMENTS).unwrap();
-
- let attributes: HumanReadableAttributeList =
- serde_json::from_value(TEST_AFFIX_JSON.clone()).unwrap();
- let attributes = attributes.into_normal().unwrap();
+ let attributes = AttributeList::parse(&TEST_AFFIX_JSON.to_string()).unwrap();
let mut expanded = WordMap::default();
@@ -167,54 +147,56 @@ mod tests {
fn plural_giants() {
let words = parse_word_list("1\ngiant/SM").unwrap();
- let attributes: HumanReadableAttributeList = serde_json::from_value(json!({
- "affixes": {
- "S": {
- "suffix": true,
- "cross_product": true,
- "replacements": [
- {
- "remove": "y",
- "add": "ies",
- "condition": "[^aeiou]"
- },
- {
- "remove": "",
- "add": "s",
- "condition": "[aeiou]y"
- },
- {
- "remove": "",
- "add": "s",
- "condition": "[^sxzhy]"
- }
- ],
- "adds_metadata": {
- "noun": {
- "is_plural": true
+ let attributes = AttributeList::parse(
+ &json!({
+ "affixes": {
+ "S": {
+ "suffix": true,
+ "cross_product": true,
+ "replacements": [
+ {
+ "remove": "y",
+ "add": "ies",
+ "condition": "[^aeiou]"
+ },
+ {
+ "remove": "",
+ "add": "s",
+ "condition": "[aeiou]y"
+ },
+ {
+ "remove": "",
+ "add": "s",
+ "condition": "[^sxzhy]"
+ }
+ ],
+ "adds_metadata": {
+ "noun": {
+ "is_plural": true
+ }
+ },
+ "gifts_metadata": {
+ "noun": {}
}
},
- "gifts_metadata": {
- "noun": {}
+ "M": {
+ "suffix": true,
+ "cross_product": true,
+ "replacements": [
+ {
+ "remove": "",
+ "add": "'s",
+ "condition": "."
+ }
+ ],
+ "adds_metadata": {},
+ "gifts_metadata": {}
}
- },
- "M": {
- "suffix": true,
- "cross_product": true,
- "replacements": [
- {
- "remove": "",
- "add": "'s",
- "condition": "."
- }
- ],
- "adds_metadata": {},
- "gifts_metadata": {}
}
- }
- }))
+ })
+ .to_string(),
+ )
.unwrap();
- let attributes = attributes.into_normal().unwrap();
let mut expanded = WordMap::default();
@@ -226,100 +208,4 @@ mod tests {
let giants_data = expanded.get_with_str("giants").unwrap();
assert!(giants_data.metadata.is_plural_noun());
}
-
- fn build_expanded() -> WordMap {
- let words = parse_default_word_list().unwrap();
- let attributes = parse_default_attribute_list();
-
- let mut expanded = WordMap::default();
-
- attributes.expand_marked_words(words, &mut expanded);
-
- expanded
- }
-
- #[test]
- fn can_expand_default() {
- build_expanded();
- }
-
- #[test]
- fn expanded_contains_giants() {
- assert!(build_expanded().contains_str("giants"));
- }
-
- #[test]
- fn expanded_contains_deallocate() {
- assert!(build_expanded().contains_str("deallocate"));
- }
-
- #[test]
- fn expanded_contains_repo() {
- let expanded = build_expanded();
-
- assert!(expanded.contains_str("repo"));
- assert!(expanded.contains_str("repos"));
- assert!(expanded.contains_str("repo's"));
- }
-
- #[test]
- fn expanded_contains_possessive_abandonment() {
- assert!(
- build_expanded()
- .get_with_str("abandonment's")
- .unwrap()
- .metadata
- .is_possessive_noun()
- )
- }
-
- #[test]
- fn has_is_not_a_nominal() {
- let expanded = build_expanded();
-
- let has = expanded.get_with_str("has");
- assert!(has.is_some());
-
- assert!(!has.unwrap().metadata.is_nominal())
- }
-
- #[test]
- fn is_is_linking_verb() {
- let expanded = build_expanded();
-
- let is = expanded.get_with_str("is");
-
- dbg!(&is);
- assert!(is.is_some());
- assert!(is.unwrap().metadata.is_linking_verb());
- }
-
- #[test]
- fn are_merged_attrs_same_as_spread_attrs() {
- let merged_word = parse_word_list("1\nblork/DGS").unwrap();
- let spread_word = parse_word_list("2\nblork/DG\nblork/S").unwrap();
-
- let merged_attrs = parse_default_attribute_list();
- let spread_attrs = parse_default_attribute_list();
-
- let mut expanded1 = WordMap::default();
- let mut expanded2 = WordMap::default();
-
- merged_attrs.expand_marked_words(merged_word, &mut expanded1);
- let expanded_merged: HashSet<String> = expanded1
- .into_iter()
- .map(|v| v.canonical_spelling.into_iter().collect())
- .collect();
-
- spread_attrs.expand_marked_words(spread_word, &mut expanded2);
- let expanded_spread: HashSet<String> = expanded2
- .into_iter()
- .map(|v| v.canonical_spelling.into_iter().collect())
- .collect();
-
- assert_eq!(
- expanded_merged.into_iter().collect::<HashSet<_>>(),
- expanded_spread.into_iter().collect::<HashSet<_>>()
- );
- }
}
diff --git a/harper-core/src/spell/word_map.rs b/harper-core/src/spell/word_map.rs
index 66a40f2f3..15378bb17 100644
--- a/harper-core/src/spell/word_map.rs
+++ b/harper-core/src/spell/word_map.rs
@@ -10,7 +10,7 @@ pub struct WordMap {
inner: HashMap<WordId, WordMapEntry>,
}
-#[derive(Debug, Clone, Eq, PartialEq)]
+#[derive(Debug, Clone, Eq, PartialEq, Hash)]
pub struct WordMapEntry {
pub metadata: WordMetadata,
pub canonical_spelling: CharString,