Offline, privacy-first grammar checker. Fast, open-source, Rust-powered
fix: we do not need a whole extra build step and crate for this
Elijah Pottera7ffd4fb00b5
diff --git a/.gitignore b/.gitignoreindex 6f572f0bc..bd44edec9 100644--- a/.gitignore+++ b/.gitignore@@ -1,3 +1,2 @@targetbuild-*.fstdiff --git a/Cargo.lock b/Cargo.lockindex b43018a8a..f257f7759 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -598,7 +598,6 @@ dependencies = ["blanket","criterion","fst",- "harper-dictionary-parsing","hashbrown 0.15.0","is-macro","itertools 0.13.0",@@ -615,18 +614,6 @@ dependencies = ["unicode-width 0.2.0",]-[[package]]-name = "harper-dictionary-parsing"-version = "0.12.0"-dependencies = [- "hashbrown 0.15.0",- "is-macro",- "serde",- "serde_json",- "smallvec",- "thiserror",-]-[[package]]name = "harper-html"version = "0.12.0"diff --git a/Cargo.toml b/Cargo.tomlindex 839007f17..9a2546b5f 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -1,5 +1,5 @@[workspace]-members = [ "harper-cli", "harper-core", "harper-dictionary-parsing", "harper-ls", "harper-comments", "harper-wasm", "harper-tree-sitter", "harper-html"]+members = [ "harper-cli", "harper-core", "harper-ls", "harper-comments", "harper-wasm", "harper-tree-sitter", "harper-html"]resolver = "2"[profile.release]diff --git a/harper-core/Cargo.toml b/harper-core/Cargo.tomlindex 7ad5162ac..daa6e9a5c 100644--- a/harper-core/Cargo.toml+++ b/harper-core/Cargo.toml@@ -7,12 +7,7 @@ license = "Apache-2.0"readme = "README.md"repository = "https://github.com/elijah-potter/harper"-[[bin]]-name = "force-build"-path = "build.rs"-[dependencies]-harper-dictionary-parsing = { path = "../harper-dictionary-parsing", version = "0.12.0" }blanket = "0.4.0"fst = "0.4.7"hashbrown = { version = "0.15.0", features = ["serde"] }@@ -33,12 +28,6 @@ levenshtein_automata = { version = "0.2.1", features = ["fst_automaton"] }criterion = { version = "0.5.1", features = ["plotters", "cargo_bench_support"], default-features = false}rand = "0.8.5"-[build-dependencies]-fst = "0.4.7"-harper-dictionary-parsing = { path = "../harper-dictionary-parsing", version = "0.12.0" }-hashbrown = { version = "0.15.0" }-itertools = "0.13.0"-[[bench]]name = "parse_demo"harness = falsediff --git a/harper-dictionary-parsing/affixes.json b/harper-core/affixes.jsonsimilarity index 100%rename from harper-dictionary-parsing/affixes.jsonrename to harper-core/affixes.jsondiff --git a/harper-core/build.rs b/harper-core/build.rsdeleted file mode 100644index 996df9bd9..000000000--- a/harper-core/build.rs+++ /dev/null@@ -1,45 +0,0 @@-use std::fs::File;-use std::io;--use fst::MapBuilder;-use harper_dictionary_parsing::{- parse_default_attribute_list, parse_default_word_list, CharString,-};-use hashbrown::HashMap;-use itertools::Itertools;--fn main() {- let wtr = io::BufWriter::new(File::create("dictionary.fst").unwrap());- let mut build = MapBuilder::new(wtr).unwrap();-- let word_list = parse_default_word_list().unwrap();- let attr_list = parse_default_attribute_list();-- // There will be at _least_ this number of words- let mut word_map = HashMap::with_capacity(word_list.len());-- attr_list.expand_marked_words(word_list, &mut word_map);-- let mut words: Vec<CharString> = word_map.iter().map(|(w, _)| w.clone()).collect();- // This may seem weird, but it ensures that the indexes from enumerate below match with those- // of the words array in FullDictionary.- words.sort_unstable();- words.dedup();- words.sort_by_key(|w| w.len()); // DO NOT MAKE THIS UNSTABLE.-- words- .iter()- .enumerate()- .sorted_by_key(|w| w.1)- .for_each(|(i, s)| {- let word = s.iter().collect::<String>();- build- .insert(word, i as u64)- .expect("Insertion not in lexicographical order!");- });-- build.finish().expect("Unable to build map of dictionary!");-- println!("cargo::rerun-if-changed=build.rs");- println!("cargo::rerun-if-changed=../harper-dictionary-parsing/dictionary.dict");-}diff --git a/harper-dictionary-parsing/dictionary.dict b/harper-core/dictionary.dictsimilarity index 100%rename from harper-dictionary-parsing/dictionary.dictrename to harper-core/dictionary.dictdiff --git a/harper-dictionary-parsing/src/char_string.rs b/harper-core/src/char_string.rssimilarity index 100%rename from harper-dictionary-parsing/src/char_string.rsrename to harper-core/src/char_string.rsdiff --git a/harper-core/src/lib.rs b/harper-core/src/lib.rsindex ac55825b7..b7f5a2c02 100644--- a/harper-core/src/lib.rs+++ b/harper-core/src/lib.rs@@ -2,6 +2,7 @@#![allow(dead_code)]mod char_ext;+mod char_string;mod document;pub mod language_detection;mod lexing;@@ -10,24 +11,26 @@ mod mask;pub mod parsers;pub mod patterns;mod punctuation;+mod span;mod spell;mod sync;mod token;mod vec_ext;+mod word_metadata;use std::collections::VecDeque;+pub use char_string::{CharString, CharStringExt};pub use document::Document;-pub use harper_dictionary_parsing::char_string::{CharString, CharStringExt};-pub use harper_dictionary_parsing::span::Span;-pub use harper_dictionary_parsing::{word_metadata::Tense, WordMetadata};use linting::Lint;pub use mask::{Mask, Masker};pub use punctuation::{Punctuation, Quote};+pub use span::Span;pub use spell::{Dictionary, FstDictionary, FullDictionary, MergedDictionary};pub use sync::Lrc;pub use token::{FatToken, Token, TokenKind, TokenStringExt};pub use vec_ext::VecExt;+pub use word_metadata::{AdverbData, ConjunctionData, NounData, Tense, VerbData, WordMetadata};/// A utility function that removes overlapping lints in a vector,/// keeping the more important ones.diff --git a/harper-dictionary-parsing/src/span.rs b/harper-core/src/span.rssimilarity index 97%rename from harper-dictionary-parsing/src/span.rsrename to harper-core/src/span.rsindex 4fdac1d08..042b910c8 100644--- a/harper-dictionary-parsing/src/span.rs+++ b/harper-core/src/span.rs@@ -11,7 +11,9 @@ pub struct Span {impl Span {pub fn new(start: usize, end: usize) -> Self {- assert!(start <= end);+ if start > end {+ panic!("{} > {}", start, end);+ }Self { start, end }}diff --git a/harper-core/src/spell/fst_dictionary.rs b/harper-core/src/spell/fst_dictionary.rsindex 9ae2dcc03..d3ac36e4a 100644--- a/harper-core/src/spell/fst_dictionary.rs+++ b/harper-core/src/spell/fst_dictionary.rs@@ -1,4 +1,7 @@-use super::{seq_to_normalized, FullDictionary};+use super::{+ hunspell::{parse_default_attribute_list, parse_default_word_list},+ seq_to_normalized, FullDictionary,+};use fst::{map::StreamWithState, IntoStreamer, Map as FstMap, Streamer};use hashbrown::HashMap;use itertools::Itertools;@@ -19,13 +22,14 @@ pub struct FstDictionary {/// The uncached function that is used to produce the original copy of the/// curated dictionary.fn uncached_inner_new() -> Arc<FstDictionary> {- let full_dict = FullDictionary::curated();- let word_map = FstMap::new(include_bytes!("../../dictionary.fst").to_vec()).unwrap();+ let word_list = parse_default_word_list().unwrap();+ let attr_list = parse_default_attribute_list();- Arc::new(FstDictionary {- full_dict,- word_map,- })+ // There will be at _least_ this number of words+ let mut word_map = HashMap::with_capacity(word_list.len());+ attr_list.expand_marked_words(word_list, &mut word_map);++ Arc::new(FstDictionary::new(word_map))}const EXPECTED_DISTANCE: u8 = 3;@@ -187,7 +191,8 @@ impl Dictionary for FstDictionary {mod tests {use itertools::Itertools;- use crate::{spell::seq_to_normalized, CharStringExt, Dictionary};+ use crate::CharStringExt;+ use crate::{spell::seq_to_normalized, Dictionary};use super::FstDictionary;@@ -197,8 +202,10 @@ mod tests {for word in dict.words_iter() {let misspelled_normalized = seq_to_normalized(word);- let misspelled_word: String = misspelled_normalized.to_string();- let misspelled_lower: String = misspelled_normalized.to_lower().to_string();+ let misspelled_word = misspelled_normalized.to_string();+ let misspelled_lower = misspelled_normalized.to_lower().to_string();++ dbg!(&misspelled_lower);assert!(!misspelled_word.is_empty());assert!(diff --git a/harper-core/src/spell/full_dictionary.rs b/harper-core/src/spell/full_dictionary.rsindex fd9fc9772..630234058 100644--- a/harper-core/src/spell/full_dictionary.rs+++ b/harper-core/src/spell/full_dictionary.rs@@ -1,4 +1,8 @@-use super::{edit_distance_min_alloc, seq_to_normalized};+use super::{+ edit_distance_min_alloc,+ hunspell::{parse_default_attribute_list, parse_default_word_list},+ seq_to_normalized,+};use hashbrown::HashMap;use itertools::Itertools;use smallvec::{SmallVec, ToSmallVec};@@ -6,7 +10,6 @@ use std::sync::Arc;use super::dictionary::Dictionary;use crate::{CharString, CharStringExt, WordMetadata};-use harper_dictionary_parsing::{parse_default_attribute_list, parse_default_word_list};/// A full, fat dictionary./// All elements are stored in-memory.@@ -248,7 +251,7 @@ impl Dictionary for FullDictionary {#[cfg(test)]mod tests {- use harper_dictionary_parsing::CharString;+ use crate::CharString;use itertools::Itertools;use crate::{Dictionary, FullDictionary};diff --git a/harper-dictionary-parsing/src/affix_replacement.rs b/harper-core/src/spell/hunspell/affix_replacement.rssimilarity index 100%rename from harper-dictionary-parsing/src/affix_replacement.rsrename to harper-core/src/spell/hunspell/affix_replacement.rsdiff --git a/harper-dictionary-parsing/src/attribute_list.rs b/harper-core/src/spell/hunspell/attribute_list.rssimilarity index 100%rename from harper-dictionary-parsing/src/attribute_list.rsrename to harper-core/src/spell/hunspell/attribute_list.rsdiff --git a/harper-dictionary-parsing/src/error.rs b/harper-core/src/spell/hunspell/error.rssimilarity index 100%rename from harper-dictionary-parsing/src/error.rsrename to harper-core/src/spell/hunspell/error.rsdiff --git a/harper-dictionary-parsing/src/expansion.rs b/harper-core/src/spell/hunspell/expansion.rssimilarity index 100%rename from harper-dictionary-parsing/src/expansion.rsrename to harper-core/src/spell/hunspell/expansion.rsdiff --git a/harper-dictionary-parsing/src/matcher.rs b/harper-core/src/spell/hunspell/matcher.rssimilarity index 98%rename from harper-dictionary-parsing/src/matcher.rsrename to harper-core/src/spell/hunspell/matcher.rsindex e9e1a17f3..f0574185f 100644--- a/harper-dictionary-parsing/src/matcher.rs+++ b/harper-core/src/spell/hunspell/matcher.rs@@ -129,7 +129,7 @@ pub enum Error {#[cfg(test)]mod tests {use super::Matcher;- use crate::matcher::Operator;+ use crate::spell::hunspell::matcher::Operator;#[test]fn parses_simple() {diff --git a/harper-dictionary-parsing/src/lib.rs b/harper-core/src/spell/hunspell/mod.rssimilarity index 94%rename from harper-dictionary-parsing/src/lib.rsrename to harper-core/src/spell/hunspell/mod.rsindex f3f132f4a..b5a3b48f3 100644--- a/harper-dictionary-parsing/src/lib.rs+++ b/harper-core/src/spell/hunspell/mod.rs@@ -1,30 +1,24 @@mod affix_replacement;mod attribute_list;-pub mod char_string;mod error;mod expansion;mod matcher;-pub mod span;-pub mod word_list;-pub mod word_metadata;+mod word_list;pub use attribute_list::AttributeList;use attribute_list::HumanReadableAttributeList;-pub use char_string::{CharString, CharStringExt};pub use error::Error;-pub use span::Span;-pub use word_metadata::WordMetadata;use self::word_list::parse_word_list;pub use self::word_list::MarkedWord;pub fn parse_default_word_list() -> Result<Vec<MarkedWord>, Error> {- parse_word_list(include_str!("../dictionary.dict"))+ parse_word_list(include_str!("../../../dictionary.dict"))}pub fn parse_default_attribute_list() -> AttributeList {let human_readable: HumanReadableAttributeList =- serde_json::from_str(include_str!("../affixes.json"))+ serde_json::from_str(include_str!("../../../affixes.json")).expect("The built-in affix list should always be valid.");human_readable@@ -39,7 +33,7 @@ mod tests {use super::word_list::parse_word_list;use super::{parse_default_attribute_list, parse_default_word_list};- use crate::attribute_list::HumanReadableAttributeList;+ use crate::spell::hunspell::attribute_list::HumanReadableAttributeList;use crate::{CharString, WordMetadata};pub const TEST_WORD_LIST: &str = "3\nhello\ntry/B\nwork/AB";diff --git a/harper-dictionary-parsing/src/word_list.rs b/harper-core/src/spell/hunspell/word_list.rssimilarity index 100%rename from harper-dictionary-parsing/src/word_list.rsrename to harper-core/src/spell/hunspell/word_list.rsdiff --git a/harper-core/src/spell/merged_dictionary.rs b/harper-core/src/spell/merged_dictionary.rsindex 6569b3ab3..717e064f3 100644--- a/harper-core/src/spell/merged_dictionary.rs+++ b/harper-core/src/spell/merged_dictionary.rs@@ -1,6 +1,7 @@-use itertools::Itertools;use std::sync::Arc;+use itertools::Itertools;+use super::dictionary::Dictionary;use crate::{CharString, WordMetadata};diff --git a/harper-core/src/token.rs b/harper-core/src/token.rsindex 74efdcc5e..ce1e70fba 100644--- a/harper-core/src/token.rs+++ b/harper-core/src/token.rs@@ -6,8 +6,8 @@ use serde::{Deserialize, Serialize};use crate::punctuation::Punctuation;use crate::Span;+use crate::{ConjunctionData, NounData};use crate::{Quote, WordMetadata};-use harper_dictionary_parsing::word_metadata::{ConjunctionData, NounData};#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Default)]pub struct Token {diff --git a/harper-dictionary-parsing/src/word_metadata.rs b/harper-core/src/word_metadata.rssimilarity index 100%rename from harper-dictionary-parsing/src/word_metadata.rsrename to harper-core/src/word_metadata.rsdiff --git a/harper-dictionary-parsing/Cargo.toml b/harper-dictionary-parsing/Cargo.tomldeleted file mode 100644index 1b6de9a07..000000000--- a/harper-dictionary-parsing/Cargo.toml+++ /dev/null@@ -1,15 +0,0 @@-[package]-name = "harper-dictionary-parsing"-version = "0.12.0"-edition = "2021"-description = "The language checker for developers."-license = "Apache-2.0"-repository = "https://github.com/elijah-potter/harper"--[dependencies]-hashbrown = { version = "0.15.0", features = ["serde"] }-is-macro = "0.3.6"-serde = { version = "1.0.209", features = ["derive"] }-serde_json = "1.0.132"-smallvec = { version = "1.13.2", features = ["serde"] }-thiserror = "1.0.65"