Skip to content

Offline, privacy-first grammar checker. Fast, open-source, Rust-powered

fix: we do not need a whole extra build step and crate for this

Elijah Pottera7ffd4fb00b5
diff --git a/.gitignore b/.gitignore
index 6f572f0bc..bd44edec9 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,3 +1,2 @@
target
build
-*.fst
diff --git a/Cargo.lock b/Cargo.lock
index b43018a8a..f257f7759 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -598,7 +598,6 @@ dependencies = [
"blanket",
"criterion",
"fst",
- "harper-dictionary-parsing",
"hashbrown 0.15.0",
"is-macro",
"itertools 0.13.0",
@@ -615,18 +614,6 @@ dependencies = [
"unicode-width 0.2.0",
]
-[[package]]
-name = "harper-dictionary-parsing"
-version = "0.12.0"
-dependencies = [
- "hashbrown 0.15.0",
- "is-macro",
- "serde",
- "serde_json",
- "smallvec",
- "thiserror",
-]
-
[[package]]
name = "harper-html"
version = "0.12.0"
diff --git a/Cargo.toml b/Cargo.toml
index 839007f17..9a2546b5f 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -1,5 +1,5 @@
[workspace]
-members = [ "harper-cli", "harper-core", "harper-dictionary-parsing", "harper-ls", "harper-comments", "harper-wasm", "harper-tree-sitter", "harper-html"]
+members = [ "harper-cli", "harper-core", "harper-ls", "harper-comments", "harper-wasm", "harper-tree-sitter", "harper-html"]
resolver = "2"
[profile.release]
diff --git a/harper-core/Cargo.toml b/harper-core/Cargo.toml
index 7ad5162ac..daa6e9a5c 100644
--- a/harper-core/Cargo.toml
+++ b/harper-core/Cargo.toml
@@ -7,12 +7,7 @@ license = "Apache-2.0"
readme = "README.md"
repository = "https://github.com/elijah-potter/harper"
-[[bin]]
-name = "force-build"
-path = "build.rs"
-
[dependencies]
-harper-dictionary-parsing = { path = "../harper-dictionary-parsing", version = "0.12.0" }
blanket = "0.4.0"
fst = "0.4.7"
hashbrown = { version = "0.15.0", features = ["serde"] }
@@ -33,12 +28,6 @@ levenshtein_automata = { version = "0.2.1", features = ["fst_automaton"] }
criterion = { version = "0.5.1", features = ["plotters", "cargo_bench_support"], default-features = false}
rand = "0.8.5"
-[build-dependencies]
-fst = "0.4.7"
-harper-dictionary-parsing = { path = "../harper-dictionary-parsing", version = "0.12.0" }
-hashbrown = { version = "0.15.0" }
-itertools = "0.13.0"
-
[[bench]]
name = "parse_demo"
harness = false
diff --git a/harper-dictionary-parsing/affixes.json b/harper-core/affixes.json
similarity index 100%
rename from harper-dictionary-parsing/affixes.json
rename to harper-core/affixes.json
diff --git a/harper-core/build.rs b/harper-core/build.rs
deleted file mode 100644
index 996df9bd9..000000000
--- a/harper-core/build.rs
+++ /dev/null
@@ -1,45 +0,0 @@
-use std::fs::File;
-use std::io;
-
-use fst::MapBuilder;
-use harper_dictionary_parsing::{
- parse_default_attribute_list, parse_default_word_list, CharString,
-};
-use hashbrown::HashMap;
-use itertools::Itertools;
-
-fn main() {
- let wtr = io::BufWriter::new(File::create("dictionary.fst").unwrap());
- let mut build = MapBuilder::new(wtr).unwrap();
-
- let word_list = parse_default_word_list().unwrap();
- let attr_list = parse_default_attribute_list();
-
- // There will be at _least_ this number of words
- let mut word_map = HashMap::with_capacity(word_list.len());
-
- attr_list.expand_marked_words(word_list, &mut word_map);
-
- let mut words: Vec<CharString> = word_map.iter().map(|(w, _)| w.clone()).collect();
- // This may seem weird, but it ensures that the indexes from enumerate below match with those
- // of the words array in FullDictionary.
- words.sort_unstable();
- words.dedup();
- words.sort_by_key(|w| w.len()); // DO NOT MAKE THIS UNSTABLE.
-
- words
- .iter()
- .enumerate()
- .sorted_by_key(|w| w.1)
- .for_each(|(i, s)| {
- let word = s.iter().collect::<String>();
- build
- .insert(word, i as u64)
- .expect("Insertion not in lexicographical order!");
- });
-
- build.finish().expect("Unable to build map of dictionary!");
-
- println!("cargo::rerun-if-changed=build.rs");
- println!("cargo::rerun-if-changed=../harper-dictionary-parsing/dictionary.dict");
-}
diff --git a/harper-dictionary-parsing/dictionary.dict b/harper-core/dictionary.dict
similarity index 100%
rename from harper-dictionary-parsing/dictionary.dict
rename to harper-core/dictionary.dict
diff --git a/harper-dictionary-parsing/src/char_string.rs b/harper-core/src/char_string.rs
similarity index 100%
rename from harper-dictionary-parsing/src/char_string.rs
rename to harper-core/src/char_string.rs
diff --git a/harper-core/src/lib.rs b/harper-core/src/lib.rs
index ac55825b7..b7f5a2c02 100644
--- a/harper-core/src/lib.rs
+++ b/harper-core/src/lib.rs
@@ -2,6 +2,7 @@
#![allow(dead_code)]
mod char_ext;
+mod char_string;
mod document;
pub mod language_detection;
mod lexing;
@@ -10,24 +11,26 @@ mod mask;
pub mod parsers;
pub mod patterns;
mod punctuation;
+mod span;
mod spell;
mod sync;
mod token;
mod vec_ext;
+mod word_metadata;
use std::collections::VecDeque;
+pub use char_string::{CharString, CharStringExt};
pub use document::Document;
-pub use harper_dictionary_parsing::char_string::{CharString, CharStringExt};
-pub use harper_dictionary_parsing::span::Span;
-pub use harper_dictionary_parsing::{word_metadata::Tense, WordMetadata};
use linting::Lint;
pub use mask::{Mask, Masker};
pub use punctuation::{Punctuation, Quote};
+pub use span::Span;
pub use spell::{Dictionary, FstDictionary, FullDictionary, MergedDictionary};
pub use sync::Lrc;
pub use token::{FatToken, Token, TokenKind, TokenStringExt};
pub use vec_ext::VecExt;
+pub use word_metadata::{AdverbData, ConjunctionData, NounData, Tense, VerbData, WordMetadata};
/// A utility function that removes overlapping lints in a vector,
/// keeping the more important ones.
diff --git a/harper-dictionary-parsing/src/span.rs b/harper-core/src/span.rs
similarity index 97%
rename from harper-dictionary-parsing/src/span.rs
rename to harper-core/src/span.rs
index 4fdac1d08..042b910c8 100644
--- a/harper-dictionary-parsing/src/span.rs
+++ b/harper-core/src/span.rs
@@ -11,7 +11,9 @@ pub struct Span {
impl Span {
pub fn new(start: usize, end: usize) -> Self {
- assert!(start <= end);
+ if start > end {
+ panic!("{} > {}", start, end);
+ }
Self { start, end }
}
diff --git a/harper-core/src/spell/fst_dictionary.rs b/harper-core/src/spell/fst_dictionary.rs
index 9ae2dcc03..d3ac36e4a 100644
--- a/harper-core/src/spell/fst_dictionary.rs
+++ b/harper-core/src/spell/fst_dictionary.rs
@@ -1,4 +1,7 @@
-use super::{seq_to_normalized, FullDictionary};
+use super::{
+ hunspell::{parse_default_attribute_list, parse_default_word_list},
+ seq_to_normalized, FullDictionary,
+};
use fst::{map::StreamWithState, IntoStreamer, Map as FstMap, Streamer};
use hashbrown::HashMap;
use itertools::Itertools;
@@ -19,13 +22,14 @@ pub struct FstDictionary {
/// The uncached function that is used to produce the original copy of the
/// curated dictionary.
fn uncached_inner_new() -> Arc<FstDictionary> {
- let full_dict = FullDictionary::curated();
- let word_map = FstMap::new(include_bytes!("../../dictionary.fst").to_vec()).unwrap();
+ let word_list = parse_default_word_list().unwrap();
+ let attr_list = parse_default_attribute_list();
- Arc::new(FstDictionary {
- full_dict,
- word_map,
- })
+ // There will be at _least_ this number of words
+ let mut word_map = HashMap::with_capacity(word_list.len());
+ attr_list.expand_marked_words(word_list, &mut word_map);
+
+ Arc::new(FstDictionary::new(word_map))
}
const EXPECTED_DISTANCE: u8 = 3;
@@ -187,7 +191,8 @@ impl Dictionary for FstDictionary {
mod tests {
use itertools::Itertools;
- use crate::{spell::seq_to_normalized, CharStringExt, Dictionary};
+ use crate::CharStringExt;
+ use crate::{spell::seq_to_normalized, Dictionary};
use super::FstDictionary;
@@ -197,8 +202,10 @@ mod tests {
for word in dict.words_iter() {
let misspelled_normalized = seq_to_normalized(word);
- let misspelled_word: String = misspelled_normalized.to_string();
- let misspelled_lower: String = misspelled_normalized.to_lower().to_string();
+ let misspelled_word = misspelled_normalized.to_string();
+ let misspelled_lower = misspelled_normalized.to_lower().to_string();
+
+ dbg!(&misspelled_lower);
assert!(!misspelled_word.is_empty());
assert!(
diff --git a/harper-core/src/spell/full_dictionary.rs b/harper-core/src/spell/full_dictionary.rs
index fd9fc9772..630234058 100644
--- a/harper-core/src/spell/full_dictionary.rs
+++ b/harper-core/src/spell/full_dictionary.rs
@@ -1,4 +1,8 @@
-use super::{edit_distance_min_alloc, seq_to_normalized};
+use super::{
+ edit_distance_min_alloc,
+ hunspell::{parse_default_attribute_list, parse_default_word_list},
+ seq_to_normalized,
+};
use hashbrown::HashMap;
use itertools::Itertools;
use smallvec::{SmallVec, ToSmallVec};
@@ -6,7 +10,6 @@ use std::sync::Arc;
use super::dictionary::Dictionary;
use crate::{CharString, CharStringExt, WordMetadata};
-use harper_dictionary_parsing::{parse_default_attribute_list, parse_default_word_list};
/// A full, fat dictionary.
/// All elements are stored in-memory.
@@ -248,7 +251,7 @@ impl Dictionary for FullDictionary {
#[cfg(test)]
mod tests {
- use harper_dictionary_parsing::CharString;
+ use crate::CharString;
use itertools::Itertools;
use crate::{Dictionary, FullDictionary};
diff --git a/harper-dictionary-parsing/src/affix_replacement.rs b/harper-core/src/spell/hunspell/affix_replacement.rs
similarity index 100%
rename from harper-dictionary-parsing/src/affix_replacement.rs
rename to harper-core/src/spell/hunspell/affix_replacement.rs
diff --git a/harper-dictionary-parsing/src/attribute_list.rs b/harper-core/src/spell/hunspell/attribute_list.rs
similarity index 100%
rename from harper-dictionary-parsing/src/attribute_list.rs
rename to harper-core/src/spell/hunspell/attribute_list.rs
diff --git a/harper-dictionary-parsing/src/error.rs b/harper-core/src/spell/hunspell/error.rs
similarity index 100%
rename from harper-dictionary-parsing/src/error.rs
rename to harper-core/src/spell/hunspell/error.rs
diff --git a/harper-dictionary-parsing/src/expansion.rs b/harper-core/src/spell/hunspell/expansion.rs
similarity index 100%
rename from harper-dictionary-parsing/src/expansion.rs
rename to harper-core/src/spell/hunspell/expansion.rs
diff --git a/harper-dictionary-parsing/src/matcher.rs b/harper-core/src/spell/hunspell/matcher.rs
similarity index 98%
rename from harper-dictionary-parsing/src/matcher.rs
rename to harper-core/src/spell/hunspell/matcher.rs
index e9e1a17f3..f0574185f 100644
--- a/harper-dictionary-parsing/src/matcher.rs
+++ b/harper-core/src/spell/hunspell/matcher.rs
@@ -129,7 +129,7 @@ pub enum Error {
#[cfg(test)]
mod tests {
use super::Matcher;
- use crate::matcher::Operator;
+ use crate::spell::hunspell::matcher::Operator;
#[test]
fn parses_simple() {
diff --git a/harper-dictionary-parsing/src/lib.rs b/harper-core/src/spell/hunspell/mod.rs
similarity index 94%
rename from harper-dictionary-parsing/src/lib.rs
rename to harper-core/src/spell/hunspell/mod.rs
index f3f132f4a..b5a3b48f3 100644
--- a/harper-dictionary-parsing/src/lib.rs
+++ b/harper-core/src/spell/hunspell/mod.rs
@@ -1,30 +1,24 @@
mod affix_replacement;
mod attribute_list;
-pub mod char_string;
mod error;
mod expansion;
mod matcher;
-pub mod span;
-pub mod word_list;
-pub mod word_metadata;
+mod word_list;
pub use attribute_list::AttributeList;
use attribute_list::HumanReadableAttributeList;
-pub use char_string::{CharString, CharStringExt};
pub use error::Error;
-pub use span::Span;
-pub use word_metadata::WordMetadata;
use self::word_list::parse_word_list;
pub use self::word_list::MarkedWord;
pub fn parse_default_word_list() -> Result<Vec<MarkedWord>, Error> {
- parse_word_list(include_str!("../dictionary.dict"))
+ parse_word_list(include_str!("../../../dictionary.dict"))
}
pub fn parse_default_attribute_list() -> AttributeList {
let human_readable: HumanReadableAttributeList =
- serde_json::from_str(include_str!("../affixes.json"))
+ serde_json::from_str(include_str!("../../../affixes.json"))
.expect("The built-in affix list should always be valid.");
human_readable
@@ -39,7 +33,7 @@ mod tests {
use super::word_list::parse_word_list;
use super::{parse_default_attribute_list, parse_default_word_list};
- use crate::attribute_list::HumanReadableAttributeList;
+ use crate::spell::hunspell::attribute_list::HumanReadableAttributeList;
use crate::{CharString, WordMetadata};
pub const TEST_WORD_LIST: &str = "3\nhello\ntry/B\nwork/AB";
diff --git a/harper-dictionary-parsing/src/word_list.rs b/harper-core/src/spell/hunspell/word_list.rs
similarity index 100%
rename from harper-dictionary-parsing/src/word_list.rs
rename to harper-core/src/spell/hunspell/word_list.rs
diff --git a/harper-core/src/spell/merged_dictionary.rs b/harper-core/src/spell/merged_dictionary.rs
index 6569b3ab3..717e064f3 100644
--- a/harper-core/src/spell/merged_dictionary.rs
+++ b/harper-core/src/spell/merged_dictionary.rs
@@ -1,6 +1,7 @@
-use itertools::Itertools;
use std::sync::Arc;
+use itertools::Itertools;
+
use super::dictionary::Dictionary;
use crate::{CharString, WordMetadata};
diff --git a/harper-core/src/token.rs b/harper-core/src/token.rs
index 74efdcc5e..ce1e70fba 100644
--- a/harper-core/src/token.rs
+++ b/harper-core/src/token.rs
@@ -6,8 +6,8 @@ use serde::{Deserialize, Serialize};
use crate::punctuation::Punctuation;
use crate::Span;
+use crate::{ConjunctionData, NounData};
use crate::{Quote, WordMetadata};
-use harper_dictionary_parsing::word_metadata::{ConjunctionData, NounData};
#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Default)]
pub struct Token {
diff --git a/harper-dictionary-parsing/src/word_metadata.rs b/harper-core/src/word_metadata.rs
similarity index 100%
rename from harper-dictionary-parsing/src/word_metadata.rs
rename to harper-core/src/word_metadata.rs
diff --git a/harper-dictionary-parsing/Cargo.toml b/harper-dictionary-parsing/Cargo.toml
deleted file mode 100644
index 1b6de9a07..000000000
--- a/harper-dictionary-parsing/Cargo.toml
+++ /dev/null
@@ -1,15 +0,0 @@
-[package]
-name = "harper-dictionary-parsing"
-version = "0.12.0"
-edition = "2021"
-description = "The language checker for developers."
-license = "Apache-2.0"
-repository = "https://github.com/elijah-potter/harper"
-
-[dependencies]
-hashbrown = { version = "0.15.0", features = ["serde"] }
-is-macro = "0.3.6"
-serde = { version = "1.0.209", features = ["derive"] }
-serde_json = "1.0.132"
-smallvec = { version = "1.13.2", features = ["serde"] }
-thiserror = "1.0.65"