Offline, privacy-first grammar checker. Fast, open-source, Rust-powered
feat(fst-dict): move hunspell parsing to new crate harper-dictionary-parsing
Grant Lemons1cd50d60e1d7
diff --git a/Cargo.lock b/Cargo.lockindex 2f15086a3..ce0d75344 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -601,6 +601,7 @@ dependencies = ["blanket","criterion","fst",+ "harper-dictionary-parsing","hashbrown 0.15.0","is-macro","itertools 0.13.0",@@ -616,6 +617,24 @@ dependencies = ["unicode-width 0.2.0",]+[[package]]+name = "harper-dictionary-parsing"+version = "0.12.0"+dependencies = [+ "blanket",+ "hashbrown 0.15.0",+ "is-macro",+ "itertools 0.13.0",+ "ordered-float",+ "paste",+ "serde",+ "serde_json",+ "smallvec",+ "thiserror",+ "unicode-blocks",+ "unicode-width 0.2.0",+]+[[package]]name = "harper-html"version = "0.12.0"diff --git a/Cargo.toml b/Cargo.tomlindex ca07b1137..35411147e 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -1,4 +1,4 @@[workspace]-members = [ "harper-cli", "harper-core", "harper-ls", "harper-comments", "harper-wasm", "harper-tree-sitter", "harper-html"]+members = [ "harper-cli", "harper-core", "harper-dictionary-parsing", "harper-ls", "harper-comments", "harper-wasm", "harper-tree-sitter", "harper-html"]resolver = "2"diff --git a/harper-core/Cargo.toml b/harper-core/Cargo.tomlindex 65344943c..72bb57d9f 100644--- a/harper-core/Cargo.toml+++ b/harper-core/Cargo.toml@@ -8,6 +8,7 @@ readme = "README.md"repository = "https://github.com/elijah-potter/harper"[dependencies]+harper-dictionary-parsing = { path = "../harper-dictionary-parsing", version = "0.12.0" }blanket = "0.4.0"fst = { version = "0.4.7", features = ["levenshtein"] }hashbrown = { version = "0.15.0", features = ["serde"] }@@ -29,6 +30,8 @@ rand = "0.8.5"[build-dependencies]fst = { version = "0.4.7", features = ["levenshtein"] }+harper-dictionary-parsing = { path = "../harper-dictionary-parsing", version = "0.12.0" }+hashbrown = { version = "0.15.0" }[[bench]]name = "parse_demo"diff --git a/harper-core/build.rs b/harper-core/build.rsindex 951992814..d3d6aad35 100644--- a/harper-core/build.rs+++ b/harper-core/build.rs@@ -2,38 +2,34 @@ use std::fs::File;use std::io;use fst::MapBuilder;--pub fn parse_word_list(source: &str) -> Vec<String> {- let mut lines = source.lines();-- let approx_item_count = lines.next().unwrap().parse().unwrap();- let mut words = Vec::with_capacity(approx_item_count);-- for line in lines {- if let Some((word, _attributes)) = line.split_once('/') {- words.push(word.chars().collect())- } else {- words.push(line.chars().collect())- }- }-- words-}+use harper_dictionary_parsing::{+ parse_default_attribute_list, parse_default_word_list, CharString,+};+use hashbrown::HashMap;fn main() {let wtr = io::BufWriter::new(File::create("dictionary.fst").unwrap());let mut build = MapBuilder::new(wtr).unwrap();- let mut word_list: Vec<String> = parse_word_list(include_str!("dictionary.dict"));- word_list.sort();- word_list.dedup();+ let word_list = parse_default_word_list().unwrap();+ let attr_list = parse_default_attribute_list();++ // There will be at _least_ this number of words+ let mut word_map = HashMap::with_capacity(word_list.len());++ attr_list.expand_marked_words(word_list, &mut word_map);++ let mut words: Vec<CharString> = word_map.iter().map(|(v, _)| v.clone()).collect();+ words.sort();+ words.dedup();// Using u64 shouldn't pose any issues since I don't think the English// language has that many words- word_list- .iter()- .enumerate()- .for_each(|(i, s)| build.insert(s, i as u64).unwrap());+ words.iter().enumerate().for_each(|(i, s)| {+ build+ .insert(s.as_slice().iter().collect::<String>(), i as u64)+ .unwrap()+ });build.finish().expect("Unable to build map of dictionary!");diff --git a/harper-core/src/lib.rs b/harper-core/src/lib.rsindex f7ae7e8f5..20e648771 100644--- a/harper-core/src/lib.rs+++ b/harper-core/src/lib.rs@@ -16,12 +16,12 @@ mod spell;mod sync;mod token;mod vec_ext;-mod word_metadata;use std::collections::VecDeque;pub use char_string::{CharString, CharStringExt};pub use document::Document;+pub use harper_dictionary_parsing::{word_metadata::Tense, WordMetadata};use linting::Lint;pub use mask::{Mask, Masker};pub use punctuation::{Punctuation, Quote};@@ -30,7 +30,6 @@ pub use spell::{Dictionary, FstDictionary, FullDictionary, MergedDictionary};pub use sync::Lrc;pub use token::{FatToken, Token, TokenKind, TokenStringExt};pub use vec_ext::VecExt;-pub use word_metadata::{Tense, WordMetadata};/// A utility function that removes overlapping lints in a vector,/// keeping the more important ones.diff --git a/harper-core/src/spell/full_dictionary.rs b/harper-core/src/spell/full_dictionary.rsindex aee09314c..ce5dbc97a 100644--- a/harper-core/src/spell/full_dictionary.rs+++ b/harper-core/src/spell/full_dictionary.rs@@ -3,8 +3,8 @@ use hashbrown::HashMap;use smallvec::{SmallVec, ToSmallVec};use super::dictionary::Dictionary;-use super::hunspell::{parse_default_attribute_list, parse_default_word_list};use crate::{CharString, Lrc, WordMetadata};+use harper_dictionary_parsing::{parse_default_attribute_list, parse_default_word_list};/// A full, fat dictionary./// All elements are stored in-memory.diff --git a/harper-core/src/spell/mod.rs b/harper-core/src/spell/mod.rsindex 4516fed37..994cfab29 100644--- a/harper-core/src/spell/mod.rs+++ b/harper-core/src/spell/mod.rs@@ -12,7 +12,6 @@ pub use self::merged_dictionary::MergedDictionary;mod dictionary;mod fst_dictionary;mod full_dictionary;-mod hunspell;mod merged_dictionary;/// Get the closest matches in the provided [`Dictionary`] and rank themdiff --git a/harper-core/src/token.rs b/harper-core/src/token.rsindex c4316ba23..239c2c175 100644--- a/harper-core/src/token.rs+++ b/harper-core/src/token.rs@@ -6,8 +6,8 @@ use serde::{Deserialize, Serialize};use crate::punctuation::Punctuation;use crate::span::Span;-use crate::word_metadata::{ConjunctionData, NounData};use crate::{Quote, WordMetadata};+use harper_dictionary_parsing::word_metadata::{ConjunctionData, NounData};#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Default)]pub struct Token {diff --git a/harper-dictionary-parsing/Cargo.toml b/harper-dictionary-parsing/Cargo.tomlnew file mode 100644index 000000000..b7feede14--- /dev/null+++ b/harper-dictionary-parsing/Cargo.toml@@ -0,0 +1,21 @@+[package]+name = "harper-dictionary-parsing"+version = "0.12.0"+edition = "2021"+description = "The language checker for developers."+license = "Apache-2.0"+repository = "https://github.com/elijah-potter/harper"++[dependencies]+blanket = "0.4.0"+hashbrown = { version = "0.15.0", features = ["serde"] }+is-macro = "0.3.6"+itertools = "0.13.0"+ordered-float = { version = "4.4.0", features = ["serde"] }+paste = "1.0.14"+serde = { version = "1.0.209", features = ["derive"] }+serde_json = "1.0.132"+smallvec = { version = "1.13.2", features = ["serde"] }+thiserror = "1.0.65"+unicode-blocks = "0.1.9"+unicode-width = "0.2.0"diff --git a/harper-core/affixes.json b/harper-dictionary-parsing/affixes.jsonsimilarity index 100%rename from harper-core/affixes.jsonrename to harper-dictionary-parsing/affixes.jsondiff --git a/harper-core/dictionary.dict b/harper-dictionary-parsing/dictionary.dictsimilarity index 100%rename from harper-core/dictionary.dictrename to harper-dictionary-parsing/dictionary.dictdiff --git a/harper-core/src/spell/hunspell/affix_replacement.rs b/harper-dictionary-parsing/src/affix_replacement.rssimilarity index 100%rename from harper-core/src/spell/hunspell/affix_replacement.rsrename to harper-dictionary-parsing/src/affix_replacement.rsdiff --git a/harper-core/src/spell/hunspell/attribute_list.rs b/harper-dictionary-parsing/src/attribute_list.rssimilarity index 100%rename from harper-core/src/spell/hunspell/attribute_list.rsrename to harper-dictionary-parsing/src/attribute_list.rsdiff --git a/harper-dictionary-parsing/src/char_string.rs b/harper-dictionary-parsing/src/char_string.rsnew file mode 100644index 000000000..c2d970334--- /dev/null+++ b/harper-dictionary-parsing/src/char_string.rs@@ -0,0 +1,5 @@+use smallvec::SmallVec;++/// A char sequence that improves cache locality.+/// Most English words are fewer than 12 characters.+pub type CharString = SmallVec<[char; 12]>;diff --git a/harper-core/src/spell/hunspell/error.rs b/harper-dictionary-parsing/src/error.rssimilarity index 100%rename from harper-core/src/spell/hunspell/error.rsrename to harper-dictionary-parsing/src/error.rsdiff --git a/harper-core/src/spell/hunspell/expansion.rs b/harper-dictionary-parsing/src/expansion.rssimilarity index 100%rename from harper-core/src/spell/hunspell/expansion.rsrename to harper-dictionary-parsing/src/expansion.rsdiff --git a/harper-core/src/spell/hunspell/mod.rs b/harper-dictionary-parsing/src/lib.rssimilarity index 95%rename from harper-core/src/spell/hunspell/mod.rsrename to harper-dictionary-parsing/src/lib.rsindex b5a3b48f3..a57f10c71 100644--- a/harper-core/src/spell/hunspell/mod.rs+++ b/harper-dictionary-parsing/src/lib.rs@@ -1,24 +1,30 @@mod affix_replacement;mod attribute_list;+pub mod char_string;mod error;mod expansion;mod matcher;-mod word_list;+mod span;+pub mod word_list;+pub mod word_metadata;pub use attribute_list::AttributeList;use attribute_list::HumanReadableAttributeList;+pub use char_string::CharString;pub use error::Error;+pub use span::Span;+pub use word_metadata::WordMetadata;use self::word_list::parse_word_list;pub use self::word_list::MarkedWord;pub fn parse_default_word_list() -> Result<Vec<MarkedWord>, Error> {- parse_word_list(include_str!("../../../dictionary.dict"))+ parse_word_list(include_str!("../dictionary.dict"))}pub fn parse_default_attribute_list() -> AttributeList {let human_readable: HumanReadableAttributeList =- serde_json::from_str(include_str!("../../../affixes.json"))+ serde_json::from_str(include_str!("../affixes.json")).expect("The built-in affix list should always be valid.");human_readable@@ -33,7 +39,7 @@ mod tests {use super::word_list::parse_word_list;use super::{parse_default_attribute_list, parse_default_word_list};- use crate::spell::hunspell::attribute_list::HumanReadableAttributeList;+ use crate::attribute_list::HumanReadableAttributeList;use crate::{CharString, WordMetadata};pub const TEST_WORD_LIST: &str = "3\nhello\ntry/B\nwork/AB";diff --git a/harper-core/src/spell/hunspell/matcher.rs b/harper-dictionary-parsing/src/matcher.rssimilarity index 98%rename from harper-core/src/spell/hunspell/matcher.rsrename to harper-dictionary-parsing/src/matcher.rsindex f0574185f..e9e1a17f3 100644--- a/harper-core/src/spell/hunspell/matcher.rs+++ b/harper-dictionary-parsing/src/matcher.rs@@ -129,7 +129,7 @@ pub enum Error {#[cfg(test)]mod tests {use super::Matcher;- use crate::spell::hunspell::matcher::Operator;+ use crate::matcher::Operator;#[test]fn parses_simple() {diff --git a/harper-dictionary-parsing/src/span.rs b/harper-dictionary-parsing/src/span.rsnew file mode 100644index 000000000..4fdac1d08--- /dev/null+++ b/harper-dictionary-parsing/src/span.rs@@ -0,0 +1,136 @@+use std::ops::Range;++use serde::{Deserialize, Serialize};++/// A window in a [`char`] sequence.+#[derive(Debug, Clone, Copy, Serialize, Deserialize, Default, PartialEq, Eq)]+pub struct Span {+ pub start: usize,+ pub end: usize,+}++impl Span {+ pub fn new(start: usize, end: usize) -> Self {+ assert!(start <= end);+ Self { start, end }+ }++ pub fn new_with_len(start: usize, len: usize) -> Self {+ Self {+ start,+ end: start + len,+ }+ }++ pub fn len(&self) -> usize {+ self.end - self.start+ }++ pub fn is_empty(&self) -> bool {+ self.len() == 0+ }++ pub fn contains(&self, idx: usize) -> bool {+ assert!(self.start <= self.end);++ self.start <= idx && idx < self.end+ }++ pub fn overlaps_with(&self, other: Self) -> bool {+ (self.start < other.end) && (other.start < self.end)+ }++ /// Get the associated content. Will return [`None`] if any aspect is+ /// invalid.+ pub fn try_get_content<'a>(&self, source: &'a [char]) -> Option<&'a [char]> {+ if (self.start > self.end) || (self.start >= source.len()) || (self.end > source.len()) {+ if self.is_empty() {+ return Some(&source[0..0]);+ }+ return None;+ }++ Some(&source[self.start..self.end])+ }++ /// Get the associated content. Will panic if any aspect is invalid.+ pub fn get_content<'a>(&self, source: &'a [char]) -> &'a [char] {+ self.try_get_content(source).unwrap()+ }++ pub fn get_content_string(&self, source: &[char]) -> String {+ String::from_iter(self.get_content(source))+ }++ pub fn set_len(&mut self, length: usize) {+ self.end = self.start + length;+ }++ pub fn with_len(&self, length: usize) -> Self {+ let mut cloned = *self;+ cloned.set_len(length);+ cloned+ }++ // Add an amount to both [`Self::start`] and [`Self::end`]+ pub fn push_by(&mut self, by: usize) {+ self.start += by;+ self.end += by;+ }++ // Subtract an amount to both [`Self::start`] and [`Self::end`]+ pub fn pull_by(&mut self, by: usize) {+ self.start -= by;+ self.end -= by;+ }++ // Add an amount to a copy of both [`Self::start`] and [`Self::end`]+ pub fn pushed_by(&self, by: usize) -> Self {+ let mut clone = *self;+ clone.start += by;+ clone.end += by;+ clone+ }++ // Subtract an amount to a copy of both [`Self::start`] and [`Self::end`]+ pub fn pulled_by(&self, by: usize) -> Self {+ let mut clone = *self;+ clone.start -= by;+ clone.end -= by;+ clone+ }++ // Add an amount a copy of both [`Self::start`] and [`Self::end`]+ pub fn with_offset(&self, by: usize) -> Self {+ let mut clone = *self;+ clone.push_by(by);+ clone+ }+}++impl From<Range<usize>> for Span {+ fn from(value: Range<usize>) -> Self {+ Self::new(value.start, value.end)+ }+}++impl From<Span> for Range<usize> {+ fn from(value: Span) -> Self {+ value.start..value.end+ }+}++#[cfg(test)]+mod tests {+ use crate::Span;++ #[test]+ fn overlaps() {+ assert!(Span::new(0, 5).overlaps_with(Span::new(3, 6)));+ assert!(Span::new(0, 5).overlaps_with(Span::new(2, 3)));+ assert!(Span::new(0, 5).overlaps_with(Span::new(4, 5)));+ assert!(Span::new(0, 5).overlaps_with(Span::new(4, 4)));++ assert!(!Span::new(0, 3).overlaps_with(Span::new(3, 5)));+ }+}diff --git a/harper-core/src/spell/hunspell/word_list.rs b/harper-dictionary-parsing/src/word_list.rssimilarity index 100%rename from harper-core/src/spell/hunspell/word_list.rsrename to harper-dictionary-parsing/src/word_list.rsdiff --git a/harper-core/src/word_metadata.rs b/harper-dictionary-parsing/src/word_metadata.rssimilarity index 100%rename from harper-core/src/word_metadata.rsrename to harper-dictionary-parsing/src/word_metadata.rsdiff --git a/harper-ls/src/backend.rs b/harper-ls/src/backend.rsindex 45c81bdb6..ed82648cc 100644--- a/harper-ls/src/backend.rs+++ b/harper-ls/src/backend.rs@@ -104,7 +104,7 @@ impl Backend {Ok(save_dict(&config.user_dict_path, dict).await?)}- async fn generate_global_dictionary(&self) -> anyhow::Result<MergedDictionary<FullDictionary>> {+ async fn generate_global_dictionary(&self) -> anyhow::Result<MergedDictionary> {let mut dict = MergedDictionary::new();dict.add_dictionary(FullDictionary::curated());let user_dict = self.load_user_dictionary().await;@@ -112,10 +112,7 @@ impl Backend {Ok(dict)}- async fn generate_file_dictionary(- &self,- url: &Url,- ) -> anyhow::Result<MergedDictionary<FullDictionary>> {+ async fn generate_file_dictionary(&self, url: &Url) -> anyhow::Result<MergedDictionary> {let (global_dictionary, file_dictionary) = tokio::join!(self.generate_global_dictionary(),self.load_file_dictionary(url)@@ -126,7 +123,7 @@ impl Backend {};let mut global_dictionary = global_dictionary?;- global_dictionary.add_dictionary(file_dictionary.into());+ global_dictionary.add_dictionary(Arc::new(file_dictionary));Ok(global_dictionary)}diff --git a/harper-ls/src/document_state.rs b/harper-ls/src/document_state.rsindex 9b825c7f7..119833262 100644--- a/harper-ls/src/document_state.rs+++ b/harper-ls/src/document_state.rs@@ -5,7 +5,7 @@ use harper_core::{Document, FullDictionary, Lrc, MergedDictionary};pub struct DocumentState {pub document: Document,pub ident_dict: Lrc<FullDictionary>,- pub dict: Lrc<MergedDictionary<FullDictionary>>,- pub linter: LintGroup<Lrc<MergedDictionary<FullDictionary>>>,+ pub dict: Lrc<MergedDictionary>,+ pub linter: LintGroup<Lrc<MergedDictionary>>,pub language_id: Option<String>,}diff --git a/harper-wasm/src/lib.rs b/harper-wasm/src/lib.rsindex 53eeffc74..7a404504c 100644--- a/harper-wasm/src/lib.rs+++ b/harper-wasm/src/lib.rs@@ -5,15 +5,15 @@ use std::sync::Mutex;use harper_core::language_detection::is_doc_likely_english;use harper_core::linting::{LintGroup, LintGroupConfig, Linter};use harper_core::parsers::{IsolateEnglish, Markdown, PlainEnglish};-use harper_core::{remove_overlaps, Document, FullDictionary, Lrc};+use harper_core::{remove_overlaps, Document, FstDictionary, Lrc};use once_cell::sync::Lazy;use wasm_bindgen::prelude::wasm_bindgen;use wasm_bindgen::JsValue;-static LINTER: Lazy<Mutex<LintGroup<Lrc<FullDictionary>>>> = Lazy::new(|| {+static LINTER: Lazy<Mutex<LintGroup<Lrc<FstDictionary>>>> = Lazy::new(|| {Mutex::new(LintGroup::new(LintGroupConfig::default(),- FullDictionary::curated(),+ FstDictionary::curated(),))});@@ -32,13 +32,13 @@ pub fn setup() {#[wasm_bindgen]pub fn is_likely_english(text: String) -> bool {let document = Document::new_plain_english_curated(&text);- is_doc_likely_english(&document, &FullDictionary::curated())+ is_doc_likely_english(&document, &FstDictionary::curated())}/// Helper method to remove non-English text from a plain English document.#[wasm_bindgen]pub fn isolate_english(text: String) -> String {- let dict = FullDictionary::curated();+ let dict = FstDictionary::curated();let document = Document::new_curated(&text,@@ -67,8 +67,7 @@ pub fn lint(text: String) -> Vec<Lint> {let source: Vec<_> = text.chars().collect();let source = Lrc::new(source);- let document =- Document::new_from_vec(source.clone(), &mut Markdown, &FullDictionary::curated());+ let document = Document::new_from_vec(source.clone(), &mut Markdown, &FstDictionary::curated());let mut lints = LINTER.lock().unwrap().lint(&document);