Skip to content

Offline, privacy-first grammar checker. Fast, open-source, Rust-powered

feat(fst-dict): move hunspell parsing to new crate harper-dictionary-parsing

Grant Lemons1cd50d60e1d7
diff --git a/Cargo.lock b/Cargo.lock
index 2f15086a3..ce0d75344 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -601,6 +601,7 @@ dependencies = [
"blanket",
"criterion",
"fst",
+ "harper-dictionary-parsing",
"hashbrown 0.15.0",
"is-macro",
"itertools 0.13.0",
@@ -616,6 +617,24 @@ dependencies = [
"unicode-width 0.2.0",
]
+[[package]]
+name = "harper-dictionary-parsing"
+version = "0.12.0"
+dependencies = [
+ "blanket",
+ "hashbrown 0.15.0",
+ "is-macro",
+ "itertools 0.13.0",
+ "ordered-float",
+ "paste",
+ "serde",
+ "serde_json",
+ "smallvec",
+ "thiserror",
+ "unicode-blocks",
+ "unicode-width 0.2.0",
+]
+
[[package]]
name = "harper-html"
version = "0.12.0"
diff --git a/Cargo.toml b/Cargo.toml
index ca07b1137..35411147e 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -1,4 +1,4 @@
[workspace]
-members = [ "harper-cli", "harper-core", "harper-ls", "harper-comments", "harper-wasm", "harper-tree-sitter", "harper-html"]
+members = [ "harper-cli", "harper-core", "harper-dictionary-parsing", "harper-ls", "harper-comments", "harper-wasm", "harper-tree-sitter", "harper-html"]
resolver = "2"
diff --git a/harper-core/Cargo.toml b/harper-core/Cargo.toml
index 65344943c..72bb57d9f 100644
--- a/harper-core/Cargo.toml
+++ b/harper-core/Cargo.toml
@@ -8,6 +8,7 @@ readme = "README.md"
repository = "https://github.com/elijah-potter/harper"
[dependencies]
+harper-dictionary-parsing = { path = "../harper-dictionary-parsing", version = "0.12.0" }
blanket = "0.4.0"
fst = { version = "0.4.7", features = ["levenshtein"] }
hashbrown = { version = "0.15.0", features = ["serde"] }
@@ -29,6 +30,8 @@ rand = "0.8.5"
[build-dependencies]
fst = { version = "0.4.7", features = ["levenshtein"] }
+harper-dictionary-parsing = { path = "../harper-dictionary-parsing", version = "0.12.0" }
+hashbrown = { version = "0.15.0" }
[[bench]]
name = "parse_demo"
diff --git a/harper-core/build.rs b/harper-core/build.rs
index 951992814..d3d6aad35 100644
--- a/harper-core/build.rs
+++ b/harper-core/build.rs
@@ -2,38 +2,34 @@ use std::fs::File;
use std::io;
use fst::MapBuilder;
-
-pub fn parse_word_list(source: &str) -> Vec<String> {
- let mut lines = source.lines();
-
- let approx_item_count = lines.next().unwrap().parse().unwrap();
- let mut words = Vec::with_capacity(approx_item_count);
-
- for line in lines {
- if let Some((word, _attributes)) = line.split_once('/') {
- words.push(word.chars().collect())
- } else {
- words.push(line.chars().collect())
- }
- }
-
- words
-}
+use harper_dictionary_parsing::{
+ parse_default_attribute_list, parse_default_word_list, CharString,
+};
+use hashbrown::HashMap;
fn main() {
let wtr = io::BufWriter::new(File::create("dictionary.fst").unwrap());
let mut build = MapBuilder::new(wtr).unwrap();
- let mut word_list: Vec<String> = parse_word_list(include_str!("dictionary.dict"));
- word_list.sort();
- word_list.dedup();
+ let word_list = parse_default_word_list().unwrap();
+ let attr_list = parse_default_attribute_list();
+
+ // There will be at _least_ this number of words
+ let mut word_map = HashMap::with_capacity(word_list.len());
+
+ attr_list.expand_marked_words(word_list, &mut word_map);
+
+ let mut words: Vec<CharString> = word_map.iter().map(|(v, _)| v.clone()).collect();
+ words.sort();
+ words.dedup();
// Using u64 shouldn't pose any issues since I don't think the English
// language has that many words
- word_list
- .iter()
- .enumerate()
- .for_each(|(i, s)| build.insert(s, i as u64).unwrap());
+ words.iter().enumerate().for_each(|(i, s)| {
+ build
+ .insert(s.as_slice().iter().collect::<String>(), i as u64)
+ .unwrap()
+ });
build.finish().expect("Unable to build map of dictionary!");
diff --git a/harper-core/src/lib.rs b/harper-core/src/lib.rs
index f7ae7e8f5..20e648771 100644
--- a/harper-core/src/lib.rs
+++ b/harper-core/src/lib.rs
@@ -16,12 +16,12 @@ mod spell;
mod sync;
mod token;
mod vec_ext;
-mod word_metadata;
use std::collections::VecDeque;
pub use char_string::{CharString, CharStringExt};
pub use document::Document;
+pub use harper_dictionary_parsing::{word_metadata::Tense, WordMetadata};
use linting::Lint;
pub use mask::{Mask, Masker};
pub use punctuation::{Punctuation, Quote};
@@ -30,7 +30,6 @@ pub use spell::{Dictionary, FstDictionary, FullDictionary, MergedDictionary};
pub use sync::Lrc;
pub use token::{FatToken, Token, TokenKind, TokenStringExt};
pub use vec_ext::VecExt;
-pub use word_metadata::{Tense, WordMetadata};
/// A utility function that removes overlapping lints in a vector,
/// keeping the more important ones.
diff --git a/harper-core/src/spell/full_dictionary.rs b/harper-core/src/spell/full_dictionary.rs
index aee09314c..ce5dbc97a 100644
--- a/harper-core/src/spell/full_dictionary.rs
+++ b/harper-core/src/spell/full_dictionary.rs
@@ -3,8 +3,8 @@ use hashbrown::HashMap;
use smallvec::{SmallVec, ToSmallVec};
use super::dictionary::Dictionary;
-use super::hunspell::{parse_default_attribute_list, parse_default_word_list};
use crate::{CharString, Lrc, WordMetadata};
+use harper_dictionary_parsing::{parse_default_attribute_list, parse_default_word_list};
/// A full, fat dictionary.
/// All elements are stored in-memory.
diff --git a/harper-core/src/spell/mod.rs b/harper-core/src/spell/mod.rs
index 4516fed37..994cfab29 100644
--- a/harper-core/src/spell/mod.rs
+++ b/harper-core/src/spell/mod.rs
@@ -12,7 +12,6 @@ pub use self::merged_dictionary::MergedDictionary;
mod dictionary;
mod fst_dictionary;
mod full_dictionary;
-mod hunspell;
mod merged_dictionary;
/// Get the closest matches in the provided [`Dictionary`] and rank them
diff --git a/harper-core/src/token.rs b/harper-core/src/token.rs
index c4316ba23..239c2c175 100644
--- a/harper-core/src/token.rs
+++ b/harper-core/src/token.rs
@@ -6,8 +6,8 @@ use serde::{Deserialize, Serialize};
use crate::punctuation::Punctuation;
use crate::span::Span;
-use crate::word_metadata::{ConjunctionData, NounData};
use crate::{Quote, WordMetadata};
+use harper_dictionary_parsing::word_metadata::{ConjunctionData, NounData};
#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Default)]
pub struct Token {
diff --git a/harper-dictionary-parsing/Cargo.toml b/harper-dictionary-parsing/Cargo.toml
new file mode 100644
index 000000000..b7feede14
--- /dev/null
+++ b/harper-dictionary-parsing/Cargo.toml
@@ -0,0 +1,21 @@
+[package]
+name = "harper-dictionary-parsing"
+version = "0.12.0"
+edition = "2021"
+description = "The language checker for developers."
+license = "Apache-2.0"
+repository = "https://github.com/elijah-potter/harper"
+
+[dependencies]
+blanket = "0.4.0"
+hashbrown = { version = "0.15.0", features = ["serde"] }
+is-macro = "0.3.6"
+itertools = "0.13.0"
+ordered-float = { version = "4.4.0", features = ["serde"] }
+paste = "1.0.14"
+serde = { version = "1.0.209", features = ["derive"] }
+serde_json = "1.0.132"
+smallvec = { version = "1.13.2", features = ["serde"] }
+thiserror = "1.0.65"
+unicode-blocks = "0.1.9"
+unicode-width = "0.2.0"
diff --git a/harper-core/affixes.json b/harper-dictionary-parsing/affixes.json
similarity index 100%
rename from harper-core/affixes.json
rename to harper-dictionary-parsing/affixes.json
diff --git a/harper-core/dictionary.dict b/harper-dictionary-parsing/dictionary.dict
similarity index 100%
rename from harper-core/dictionary.dict
rename to harper-dictionary-parsing/dictionary.dict
diff --git a/harper-core/src/spell/hunspell/affix_replacement.rs b/harper-dictionary-parsing/src/affix_replacement.rs
similarity index 100%
rename from harper-core/src/spell/hunspell/affix_replacement.rs
rename to harper-dictionary-parsing/src/affix_replacement.rs
diff --git a/harper-core/src/spell/hunspell/attribute_list.rs b/harper-dictionary-parsing/src/attribute_list.rs
similarity index 100%
rename from harper-core/src/spell/hunspell/attribute_list.rs
rename to harper-dictionary-parsing/src/attribute_list.rs
diff --git a/harper-dictionary-parsing/src/char_string.rs b/harper-dictionary-parsing/src/char_string.rs
new file mode 100644
index 000000000..c2d970334
--- /dev/null
+++ b/harper-dictionary-parsing/src/char_string.rs
@@ -0,0 +1,5 @@
+use smallvec::SmallVec;
+
+/// A char sequence that improves cache locality.
+/// Most English words are fewer than 12 characters.
+pub type CharString = SmallVec<[char; 12]>;
diff --git a/harper-core/src/spell/hunspell/error.rs b/harper-dictionary-parsing/src/error.rs
similarity index 100%
rename from harper-core/src/spell/hunspell/error.rs
rename to harper-dictionary-parsing/src/error.rs
diff --git a/harper-core/src/spell/hunspell/expansion.rs b/harper-dictionary-parsing/src/expansion.rs
similarity index 100%
rename from harper-core/src/spell/hunspell/expansion.rs
rename to harper-dictionary-parsing/src/expansion.rs
diff --git a/harper-core/src/spell/hunspell/mod.rs b/harper-dictionary-parsing/src/lib.rs
similarity index 95%
rename from harper-core/src/spell/hunspell/mod.rs
rename to harper-dictionary-parsing/src/lib.rs
index b5a3b48f3..a57f10c71 100644
--- a/harper-core/src/spell/hunspell/mod.rs
+++ b/harper-dictionary-parsing/src/lib.rs
@@ -1,24 +1,30 @@
mod affix_replacement;
mod attribute_list;
+pub mod char_string;
mod error;
mod expansion;
mod matcher;
-mod word_list;
+mod span;
+pub mod word_list;
+pub mod word_metadata;
pub use attribute_list::AttributeList;
use attribute_list::HumanReadableAttributeList;
+pub use char_string::CharString;
pub use error::Error;
+pub use span::Span;
+pub use word_metadata::WordMetadata;
use self::word_list::parse_word_list;
pub use self::word_list::MarkedWord;
pub fn parse_default_word_list() -> Result<Vec<MarkedWord>, Error> {
- parse_word_list(include_str!("../../../dictionary.dict"))
+ parse_word_list(include_str!("../dictionary.dict"))
}
pub fn parse_default_attribute_list() -> AttributeList {
let human_readable: HumanReadableAttributeList =
- serde_json::from_str(include_str!("../../../affixes.json"))
+ serde_json::from_str(include_str!("../affixes.json"))
.expect("The built-in affix list should always be valid.");
human_readable
@@ -33,7 +39,7 @@ mod tests {
use super::word_list::parse_word_list;
use super::{parse_default_attribute_list, parse_default_word_list};
- use crate::spell::hunspell::attribute_list::HumanReadableAttributeList;
+ use crate::attribute_list::HumanReadableAttributeList;
use crate::{CharString, WordMetadata};
pub const TEST_WORD_LIST: &str = "3\nhello\ntry/B\nwork/AB";
diff --git a/harper-core/src/spell/hunspell/matcher.rs b/harper-dictionary-parsing/src/matcher.rs
similarity index 98%
rename from harper-core/src/spell/hunspell/matcher.rs
rename to harper-dictionary-parsing/src/matcher.rs
index f0574185f..e9e1a17f3 100644
--- a/harper-core/src/spell/hunspell/matcher.rs
+++ b/harper-dictionary-parsing/src/matcher.rs
@@ -129,7 +129,7 @@ pub enum Error {
#[cfg(test)]
mod tests {
use super::Matcher;
- use crate::spell::hunspell::matcher::Operator;
+ use crate::matcher::Operator;
#[test]
fn parses_simple() {
diff --git a/harper-dictionary-parsing/src/span.rs b/harper-dictionary-parsing/src/span.rs
new file mode 100644
index 000000000..4fdac1d08
--- /dev/null
+++ b/harper-dictionary-parsing/src/span.rs
@@ -0,0 +1,136 @@
+use std::ops::Range;
+
+use serde::{Deserialize, Serialize};
+
+/// A window in a [`char`] sequence.
+#[derive(Debug, Clone, Copy, Serialize, Deserialize, Default, PartialEq, Eq)]
+pub struct Span {
+ pub start: usize,
+ pub end: usize,
+}
+
+impl Span {
+ pub fn new(start: usize, end: usize) -> Self {
+ assert!(start <= end);
+ Self { start, end }
+ }
+
+ pub fn new_with_len(start: usize, len: usize) -> Self {
+ Self {
+ start,
+ end: start + len,
+ }
+ }
+
+ pub fn len(&self) -> usize {
+ self.end - self.start
+ }
+
+ pub fn is_empty(&self) -> bool {
+ self.len() == 0
+ }
+
+ pub fn contains(&self, idx: usize) -> bool {
+ assert!(self.start <= self.end);
+
+ self.start <= idx && idx < self.end
+ }
+
+ pub fn overlaps_with(&self, other: Self) -> bool {
+ (self.start < other.end) && (other.start < self.end)
+ }
+
+ /// Get the associated content. Will return [`None`] if any aspect is
+ /// invalid.
+ pub fn try_get_content<'a>(&self, source: &'a [char]) -> Option<&'a [char]> {
+ if (self.start > self.end) || (self.start >= source.len()) || (self.end > source.len()) {
+ if self.is_empty() {
+ return Some(&source[0..0]);
+ }
+ return None;
+ }
+
+ Some(&source[self.start..self.end])
+ }
+
+ /// Get the associated content. Will panic if any aspect is invalid.
+ pub fn get_content<'a>(&self, source: &'a [char]) -> &'a [char] {
+ self.try_get_content(source).unwrap()
+ }
+
+ pub fn get_content_string(&self, source: &[char]) -> String {
+ String::from_iter(self.get_content(source))
+ }
+
+ pub fn set_len(&mut self, length: usize) {
+ self.end = self.start + length;
+ }
+
+ pub fn with_len(&self, length: usize) -> Self {
+ let mut cloned = *self;
+ cloned.set_len(length);
+ cloned
+ }
+
+ // Add an amount to both [`Self::start`] and [`Self::end`]
+ pub fn push_by(&mut self, by: usize) {
+ self.start += by;
+ self.end += by;
+ }
+
+ // Subtract an amount to both [`Self::start`] and [`Self::end`]
+ pub fn pull_by(&mut self, by: usize) {
+ self.start -= by;
+ self.end -= by;
+ }
+
+ // Add an amount to a copy of both [`Self::start`] and [`Self::end`]
+ pub fn pushed_by(&self, by: usize) -> Self {
+ let mut clone = *self;
+ clone.start += by;
+ clone.end += by;
+ clone
+ }
+
+ // Subtract an amount to a copy of both [`Self::start`] and [`Self::end`]
+ pub fn pulled_by(&self, by: usize) -> Self {
+ let mut clone = *self;
+ clone.start -= by;
+ clone.end -= by;
+ clone
+ }
+
+ // Add an amount a copy of both [`Self::start`] and [`Self::end`]
+ pub fn with_offset(&self, by: usize) -> Self {
+ let mut clone = *self;
+ clone.push_by(by);
+ clone
+ }
+}
+
+impl From<Range<usize>> for Span {
+ fn from(value: Range<usize>) -> Self {
+ Self::new(value.start, value.end)
+ }
+}
+
+impl From<Span> for Range<usize> {
+ fn from(value: Span) -> Self {
+ value.start..value.end
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use crate::Span;
+
+ #[test]
+ fn overlaps() {
+ assert!(Span::new(0, 5).overlaps_with(Span::new(3, 6)));
+ assert!(Span::new(0, 5).overlaps_with(Span::new(2, 3)));
+ assert!(Span::new(0, 5).overlaps_with(Span::new(4, 5)));
+ assert!(Span::new(0, 5).overlaps_with(Span::new(4, 4)));
+
+ assert!(!Span::new(0, 3).overlaps_with(Span::new(3, 5)));
+ }
+}
diff --git a/harper-core/src/spell/hunspell/word_list.rs b/harper-dictionary-parsing/src/word_list.rs
similarity index 100%
rename from harper-core/src/spell/hunspell/word_list.rs
rename to harper-dictionary-parsing/src/word_list.rs
diff --git a/harper-core/src/word_metadata.rs b/harper-dictionary-parsing/src/word_metadata.rs
similarity index 100%
rename from harper-core/src/word_metadata.rs
rename to harper-dictionary-parsing/src/word_metadata.rs
diff --git a/harper-ls/src/backend.rs b/harper-ls/src/backend.rs
index 45c81bdb6..ed82648cc 100644
--- a/harper-ls/src/backend.rs
+++ b/harper-ls/src/backend.rs
@@ -104,7 +104,7 @@ impl Backend {
Ok(save_dict(&config.user_dict_path, dict).await?)
}
- async fn generate_global_dictionary(&self) -> anyhow::Result<MergedDictionary<FullDictionary>> {
+ async fn generate_global_dictionary(&self) -> anyhow::Result<MergedDictionary> {
let mut dict = MergedDictionary::new();
dict.add_dictionary(FullDictionary::curated());
let user_dict = self.load_user_dictionary().await;
@@ -112,10 +112,7 @@ impl Backend {
Ok(dict)
}
- async fn generate_file_dictionary(
- &self,
- url: &Url,
- ) -> anyhow::Result<MergedDictionary<FullDictionary>> {
+ async fn generate_file_dictionary(&self, url: &Url) -> anyhow::Result<MergedDictionary> {
let (global_dictionary, file_dictionary) = tokio::join!(
self.generate_global_dictionary(),
self.load_file_dictionary(url)
@@ -126,7 +123,7 @@ impl Backend {
};
let mut global_dictionary = global_dictionary?;
- global_dictionary.add_dictionary(file_dictionary.into());
+ global_dictionary.add_dictionary(Arc::new(file_dictionary));
Ok(global_dictionary)
}
diff --git a/harper-ls/src/document_state.rs b/harper-ls/src/document_state.rs
index 9b825c7f7..119833262 100644
--- a/harper-ls/src/document_state.rs
+++ b/harper-ls/src/document_state.rs
@@ -5,7 +5,7 @@ use harper_core::{Document, FullDictionary, Lrc, MergedDictionary};
pub struct DocumentState {
pub document: Document,
pub ident_dict: Lrc<FullDictionary>,
- pub dict: Lrc<MergedDictionary<FullDictionary>>,
- pub linter: LintGroup<Lrc<MergedDictionary<FullDictionary>>>,
+ pub dict: Lrc<MergedDictionary>,
+ pub linter: LintGroup<Lrc<MergedDictionary>>,
pub language_id: Option<String>,
}
diff --git a/harper-wasm/src/lib.rs b/harper-wasm/src/lib.rs
index 53eeffc74..7a404504c 100644
--- a/harper-wasm/src/lib.rs
+++ b/harper-wasm/src/lib.rs
@@ -5,15 +5,15 @@ use std::sync::Mutex;
use harper_core::language_detection::is_doc_likely_english;
use harper_core::linting::{LintGroup, LintGroupConfig, Linter};
use harper_core::parsers::{IsolateEnglish, Markdown, PlainEnglish};
-use harper_core::{remove_overlaps, Document, FullDictionary, Lrc};
+use harper_core::{remove_overlaps, Document, FstDictionary, Lrc};
use once_cell::sync::Lazy;
use wasm_bindgen::prelude::wasm_bindgen;
use wasm_bindgen::JsValue;
-static LINTER: Lazy<Mutex<LintGroup<Lrc<FullDictionary>>>> = Lazy::new(|| {
+static LINTER: Lazy<Mutex<LintGroup<Lrc<FstDictionary>>>> = Lazy::new(|| {
Mutex::new(LintGroup::new(
LintGroupConfig::default(),
- FullDictionary::curated(),
+ FstDictionary::curated(),
))
});
@@ -32,13 +32,13 @@ pub fn setup() {
#[wasm_bindgen]
pub fn is_likely_english(text: String) -> bool {
let document = Document::new_plain_english_curated(&text);
- is_doc_likely_english(&document, &FullDictionary::curated())
+ is_doc_likely_english(&document, &FstDictionary::curated())
}
/// Helper method to remove non-English text from a plain English document.
#[wasm_bindgen]
pub fn isolate_english(text: String) -> String {
- let dict = FullDictionary::curated();
+ let dict = FstDictionary::curated();
let document = Document::new_curated(
&text,
@@ -67,8 +67,7 @@ pub fn lint(text: String) -> Vec<Lint> {
let source: Vec<_> = text.chars().collect();
let source = Lrc::new(source);
- let document =
- Document::new_from_vec(source.clone(), &mut Markdown, &FullDictionary::curated());
+ let document = Document::new_from_vec(source.clone(), &mut Markdown, &FstDictionary::curated());
let mut lints = LINTER.lock().unwrap().lint(&document);