Offline, privacy-first grammar checker. Fast, open-source, Rust-powered
refactor(core): made `MutableDictionary` buildable from Rune files
Elijah Potter6f54df6f69c5
diff --git a/harper-core/src/ignored_lints/mod.rs b/harper-core/src/ignored_lints/mod.rsindex 834e19a82..84dc8337a 100644--- a/harper-core/src/ignored_lints/mod.rs+++ b/harper-core/src/ignored_lints/mod.rs@@ -9,6 +9,9 @@ use serde::{Deserialize, Serialize};use crate::{Document, linting::Lint};/// A structure that keeps track of lints that have been ignored by users.+///+/// To use this structure, apply [`Self::remove_ignored`] on the output of a+/// [`Linter`](crate::linting::Linter).#[derive(Debug, Default, Serialize, Deserialize)]pub struct IgnoredLints {context_hashes: HashSet<u64>,diff --git a/harper-core/src/spell/mutable_dictionary.rs b/harper-core/src/spell/mutable_dictionary.rsindex 19323a66f..bbdcf514b 100644--- a/harper-core/src/spell/mutable_dictionary.rs+++ b/harper-core/src/spell/mutable_dictionary.rs@@ -1,6 +1,6 @@use super::{FstDictionary, WordId,- rune::{parse_default_attribute_list, parse_default_word_list},+ rune::{self, AttributeList, parse_word_list},word_map::{WordMap, WordMapEntry},};use crate::edit_distance::edit_distance_min_alloc;@@ -30,15 +30,13 @@ pub struct MutableDictionary {/// The uncached function that is used to produce the original copy of the/// curated dictionary.fn uncached_inner_new() -> Arc<MutableDictionary> {- let word_list = parse_default_word_list().unwrap();- let attr_list = parse_default_attribute_list();-- // There will be at _least_ this number of words- let mut word_map = WordMap::default();-- attr_list.expand_marked_words(word_list, &mut word_map);-- Arc::new(MutableDictionary { word_map })+ Arc::new(+ MutableDictionary::from_rune_files(+ include_str!("../../dictionary.dict"),+ include_str!("../../affixes.json"),+ )+ .expect("Curated dictionary should be valid."),+ )}lazy_static! {@@ -52,6 +50,18 @@ impl MutableDictionary {}}+ pub fn from_rune_files(word_list: &str, attr_list: &str) -> Result<Self, rune::Error> {+ let word_list = parse_word_list(word_list)?;+ let attr_list = AttributeList::parse(attr_list)?;++ // There will be at _least_ this number of words+ let mut word_map = WordMap::default();++ attr_list.expand_marked_words(word_list, &mut word_map);++ Ok(Self { word_map })+ }+/// Create a dictionary from the curated dictionary included/// in the Harper binary./// Consider using [`super::FstDictionary::curated()`] instead, as it is more performant for spellchecking.@@ -239,6 +249,7 @@ impl From<MutableDictionary> for FstDictionary {#[cfg(test)]mod tests {+ use hashbrown::HashSet;use itertools::Itertools;use crate::{Dictionary, MutableDictionary};@@ -313,4 +324,67 @@ mod tests {assert!(!dict.get_word_metadata_str("there").unwrap().is_nominal());assert!(!dict.get_word_metadata_str("there").unwrap().is_pronoun());}++ #[test]+ fn expanded_contains_giants() {+ assert!(MutableDictionary::curated().contains_word_str("giants"));+ }++ #[test]+ fn expanded_contains_deallocate() {+ assert!(MutableDictionary::curated().contains_word_str("deallocate"));+ }++ #[test]+ fn curated_contains_repo() {+ let dict = MutableDictionary::curated();++ assert!(dict.contains_word_str("repo"));+ assert!(dict.contains_word_str("repos"));+ assert!(dict.contains_word_str("repo's"));+ }++ #[test]+ fn curated_contains_possessive_abandonment() {+ assert!(+ MutableDictionary::curated()+ .get_word_metadata_str("abandonment's")+ .unwrap()+ .is_possessive_noun()+ )+ }++ #[test]+ fn has_is_not_a_nominal() {+ let dict = MutableDictionary::curated();++ let has = dict.get_word_metadata_str("has");+ assert!(has.is_some());++ assert!(!has.unwrap().is_nominal())+ }++ #[test]+ fn is_is_linking_verb() {+ let dict = MutableDictionary::curated();++ let is = dict.get_word_metadata_str("is");++ assert!(is.is_some());+ assert!(is.unwrap().is_linking_verb());+ }++ #[test]+ fn are_merged_attrs_same_as_spread_attrs() {+ let curated_attr_list = include_str!("../../affixes.json");++ let merged = MutableDictionary::from_rune_files("1\nblork/DGS", curated_attr_list).unwrap();+ let spread =+ MutableDictionary::from_rune_files("2\nblork/DG\nblork/S", curated_attr_list).unwrap();++ assert_eq!(+ merged.word_map.into_iter().collect::<HashSet<_>>(),+ spread.word_map.into_iter().collect::<HashSet<_>>()+ );+ }}diff --git a/harper-core/src/spell/rune/attribute_list.rs b/harper-core/src/spell/rune/attribute_list.rsindex 9cf8b2d4b..14d199ed9 100644--- a/harper-core/src/spell/rune/attribute_list.rs+++ b/harper-core/src/spell/rune/attribute_list.rs@@ -16,7 +16,7 @@ pub struct AttributeList {}impl AttributeList {- pub fn into_human_readable(self) -> HumanReadableAttributeList {+ fn into_human_readable(self) -> HumanReadableAttributeList {HumanReadableAttributeList {affixes: self.affixes@@ -26,6 +26,13 @@ impl AttributeList {}}+ pub fn parse(source: &str) -> Result<Self, Error> {+ let human_readable: HumanReadableAttributeList =+ serde_json::from_str(source).map_err(|_| Error::MalformedJSON)?;++ human_readable.into_normal()+ }+/// Expand [`MarkedWord`] into a list of full words, including itself.////// Will append to the given `dest`;diff --git a/harper-core/src/spell/rune/error.rs b/harper-core/src/spell/rune/error.rsindex 8f1e113d2..41233d86a 100644--- a/harper-core/src/spell/rune/error.rs+++ b/harper-core/src/spell/rune/error.rs@@ -12,6 +12,8 @@ pub enum Error {ExpectedUnsignedInteger,#[error("Could not parse because we encountered the end of the line.")]UnexpectedEndOfLine,+ #[error("Received malformed JSON")]+ MalformedJSON,#[error("An error occured with a condition: {0}")]Matcher(#[from] matcher::Error),}diff --git a/harper-core/src/spell/rune/mod.rs b/harper-core/src/spell/rune/mod.rsindex d8e332511..0a4badbc2 100644--- a/harper-core/src/spell/rune/mod.rs+++ b/harper-core/src/spell/rune/mod.rs@@ -6,25 +6,9 @@ mod matcher;pub mod word_list;pub use attribute_list::AttributeList;-use attribute_list::HumanReadableAttributeList;pub use error::Error;-pub use self::word_list::MarkedWord;-use self::word_list::parse_word_list;--pub fn parse_default_word_list() -> Result<Vec<MarkedWord>, Error> {- parse_word_list(include_str!("../../../dictionary.dict"))-}--pub fn parse_default_attribute_list() -> AttributeList {- let human_readable: HumanReadableAttributeList =- serde_json::from_str(include_str!("../../../affixes.json"))- .expect("The built-in affix list should always be valid.");-- human_readable- .into_normal()- .expect("All expressions in the built-in attribute list should be valid.")-}+pub use self::word_list::parse_word_list;#[cfg(test)]mod tests {@@ -33,10 +17,9 @@ mod tests {use serde_json::json;use super::super::word_map::WordMap;- use super::HumanReadableAttributeList;use super::word_list::parse_word_list;- use super::{parse_default_attribute_list, parse_default_word_list};use crate::CharStringExt;+ use crate::spell::rune::AttributeList;pub const TEST_WORD_LIST: &str = "4\nhello\ntry/B\nwork/AB\nblank/";@@ -93,12 +76,12 @@ mod tests {fn assert_expansion_results(test_word_list: &str, expected: Vec<&str>) {let words = parse_word_list(test_word_list).unwrap();- let attributes: HumanReadableAttributeList =- serde_json::from_value(TEST_AFFIX_JSON.clone()).unwrap();- let attributes = attributes.into_normal().unwrap();+ let attributes = AttributeList::parse(&TEST_AFFIX_JSON.to_string()).unwrap();let mut expanded = WordMap::default();+attributes.expand_marked_words(words, &mut expanded);+let expanded: HashSet<String> = expanded.into_iter().map(|v| v.canonical_spelling.into_iter().collect())@@ -139,10 +122,7 @@ mod tests {#[test]fn correctly_expands_test_files_with_comments() {let words = parse_word_list(TEST_WORD_LIST_WITH_COMMENTS).unwrap();-- let attributes: HumanReadableAttributeList =- serde_json::from_value(TEST_AFFIX_JSON.clone()).unwrap();- let attributes = attributes.into_normal().unwrap();+ let attributes = AttributeList::parse(&TEST_AFFIX_JSON.to_string()).unwrap();let mut expanded = WordMap::default();@@ -167,54 +147,56 @@ mod tests {fn plural_giants() {let words = parse_word_list("1\ngiant/SM").unwrap();- let attributes: HumanReadableAttributeList = serde_json::from_value(json!({- "affixes": {- "S": {- "suffix": true,- "cross_product": true,- "replacements": [- {- "remove": "y",- "add": "ies",- "condition": "[^aeiou]"- },- {- "remove": "",- "add": "s",- "condition": "[aeiou]y"- },- {- "remove": "",- "add": "s",- "condition": "[^sxzhy]"- }- ],- "adds_metadata": {- "noun": {- "is_plural": true+ let attributes = AttributeList::parse(+ &json!({+ "affixes": {+ "S": {+ "suffix": true,+ "cross_product": true,+ "replacements": [+ {+ "remove": "y",+ "add": "ies",+ "condition": "[^aeiou]"+ },+ {+ "remove": "",+ "add": "s",+ "condition": "[aeiou]y"+ },+ {+ "remove": "",+ "add": "s",+ "condition": "[^sxzhy]"+ }+ ],+ "adds_metadata": {+ "noun": {+ "is_plural": true+ }+ },+ "gifts_metadata": {+ "noun": {}}},- "gifts_metadata": {- "noun": {}+ "M": {+ "suffix": true,+ "cross_product": true,+ "replacements": [+ {+ "remove": "",+ "add": "'s",+ "condition": "."+ }+ ],+ "adds_metadata": {},+ "gifts_metadata": {}}- },- "M": {- "suffix": true,- "cross_product": true,- "replacements": [- {- "remove": "",- "add": "'s",- "condition": "."- }- ],- "adds_metadata": {},- "gifts_metadata": {}}- }- }))+ })+ .to_string(),+ ).unwrap();- let attributes = attributes.into_normal().unwrap();let mut expanded = WordMap::default();@@ -226,100 +208,4 @@ mod tests {let giants_data = expanded.get_with_str("giants").unwrap();assert!(giants_data.metadata.is_plural_noun());}-- fn build_expanded() -> WordMap {- let words = parse_default_word_list().unwrap();- let attributes = parse_default_attribute_list();-- let mut expanded = WordMap::default();-- attributes.expand_marked_words(words, &mut expanded);-- expanded- }-- #[test]- fn can_expand_default() {- build_expanded();- }-- #[test]- fn expanded_contains_giants() {- assert!(build_expanded().contains_str("giants"));- }-- #[test]- fn expanded_contains_deallocate() {- assert!(build_expanded().contains_str("deallocate"));- }-- #[test]- fn expanded_contains_repo() {- let expanded = build_expanded();-- assert!(expanded.contains_str("repo"));- assert!(expanded.contains_str("repos"));- assert!(expanded.contains_str("repo's"));- }-- #[test]- fn expanded_contains_possessive_abandonment() {- assert!(- build_expanded()- .get_with_str("abandonment's")- .unwrap()- .metadata- .is_possessive_noun()- )- }-- #[test]- fn has_is_not_a_nominal() {- let expanded = build_expanded();-- let has = expanded.get_with_str("has");- assert!(has.is_some());-- assert!(!has.unwrap().metadata.is_nominal())- }-- #[test]- fn is_is_linking_verb() {- let expanded = build_expanded();-- let is = expanded.get_with_str("is");-- dbg!(&is);- assert!(is.is_some());- assert!(is.unwrap().metadata.is_linking_verb());- }-- #[test]- fn are_merged_attrs_same_as_spread_attrs() {- let merged_word = parse_word_list("1\nblork/DGS").unwrap();- let spread_word = parse_word_list("2\nblork/DG\nblork/S").unwrap();-- let merged_attrs = parse_default_attribute_list();- let spread_attrs = parse_default_attribute_list();-- let mut expanded1 = WordMap::default();- let mut expanded2 = WordMap::default();-- merged_attrs.expand_marked_words(merged_word, &mut expanded1);- let expanded_merged: HashSet<String> = expanded1- .into_iter()- .map(|v| v.canonical_spelling.into_iter().collect())- .collect();-- spread_attrs.expand_marked_words(spread_word, &mut expanded2);- let expanded_spread: HashSet<String> = expanded2- .into_iter()- .map(|v| v.canonical_spelling.into_iter().collect())- .collect();-- assert_eq!(- expanded_merged.into_iter().collect::<HashSet<_>>(),- expanded_spread.into_iter().collect::<HashSet<_>>()- );- }}diff --git a/harper-core/src/spell/word_map.rs b/harper-core/src/spell/word_map.rsindex 66a40f2f3..15378bb17 100644--- a/harper-core/src/spell/word_map.rs+++ b/harper-core/src/spell/word_map.rs@@ -10,7 +10,7 @@ pub struct WordMap {inner: HashMap<WordId, WordMapEntry>,}-#[derive(Debug, Clone, Eq, PartialEq)]+#[derive(Debug, Clone, Eq, PartialEq, Hash)]pub struct WordMapEntry {pub metadata: WordMetadata,pub canonical_spelling: CharString,