Offline, privacy-first grammar checker. Fast, open-source, Rust-powered
Switched away from build script - it was just too complex
Elijah Potter0a397bdec599
diff --git a/Cargo.lock b/Cargo.lockindex f68ffc48a..773e233d9 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -518,7 +518,6 @@ version = "0.1.0"dependencies = ["criterion","is-macro",- "regex","serde",]diff --git a/lt-core/Cargo.toml b/lt-core/Cargo.tomlindex 2c03d394b..7dd697de6 100644--- a/lt-core/Cargo.toml+++ b/lt-core/Cargo.toml@@ -2,15 +2,11 @@name = "lt-core"version = "0.1.0"edition = "2021"-build = "build.rs"[dependencies]is-macro = "0.3.0"serde = { version = "1.0.189", features = ["derive"] }-[build-dependencies]-regex = "1.10.2"-[dev-dependencies]criterion = { version = "0.5.1", features = ["html_reports"] }diff --git a/lt-core/benches/spellcheck.rs b/lt-core/benches/spellcheck.rsindex af6a29dc7..3eae41552 100644--- a/lt-core/benches/spellcheck.rs+++ b/lt-core/benches/spellcheck.rs@@ -1,12 +1,14 @@use criterion::{black_box, criterion_group, criterion_main, Criterion};-use lt_core::suggest_correct_spelling_str;+use lt_core::{suggest_correct_spelling_str, Dictionary};-fn spellcheck() {- suggest_correct_spelling_str("hello", 5, 3);+fn spellcheck(dictionary: &Dictionary) {+ suggest_correct_spelling_str("hello", 5, 3, dictionary);}fn criterion_benchmark(c: &mut Criterion) {- c.bench_function("hello 5", |b| b.iter(|| spellcheck()));+ let dictionary = Dictionary::create_from_static();++ c.bench_function("hello 5", |b| b.iter(|| spellcheck(&dictionary)));}criterion_group!(benches, criterion_benchmark);diff --git a/lt-core/build.rs b/lt-core/build.rsdeleted file mode 100644index 39b2c5d24..000000000--- a/lt-core/build.rs+++ /dev/null@@ -1,36 +0,0 @@-use regex::Regex;-use std::io::Write;-use std::{env, fs::File, io::BufWriter, path::Path};--// Generate the code needed to embed the word list as a static array to be included in the binary.-fn main() {- let english_words_raw = include_str!("../english_words.txt").replace('\r', "");-- let re = Regex::new(r"[\r-']").unwrap();- let cleaned = re.replace_all(&english_words_raw, "");-- let words: Vec<_> = cleaned- .split('\n')- .filter(|word| !word.is_empty())- .collect();-- let path = Path::new(&env::var("OUT_DIR").unwrap()).join("english.rs");- let mut out_file = BufWriter::new(File::create(path).unwrap());-- write!(- out_file,- "#[allow(clippy::all)] static ENGLISH_WORDS: [&[char]; {}] = [",- words.len()- )- .unwrap();-- for word in words {- write!(out_file, "&[").unwrap();- for character in word.chars() {- write!(out_file, "'{}', ", character).unwrap();- }- write!(out_file, "],").unwrap();- }-- write!(out_file, "];").unwrap();-}diff --git a/lt-core/src/lib.rs b/lt-core/src/lib.rsindex cc2917ac3..3004e0757 100644--- a/lt-core/src/lib.rs+++ b/lt-core/src/lib.rs@@ -3,9 +3,9 @@ mod lint;mod span;mod spell;mod token;-mod words;pub use lexer::{lex_to_end, lex_to_end_str};pub use lint::{Lint, LintKind};+pub use spell::Dictionary;pub use spell::{suggest_correct_spelling, suggest_correct_spelling_str};pub use token::{Token, TokenKind};diff --git a/lt-core/src/spell/dictionary.rs b/lt-core/src/spell/dictionary.rsnew file mode 100644index 000000000..e43bc14a7--- /dev/null+++ b/lt-core/src/spell/dictionary.rs@@ -0,0 +1,35 @@+use std::collections::HashSet;++pub struct Dictionary {+ /// Storing a separate [Vec] for iterations speeds up spellchecking by ~16% at the cost of+ /// additional memory.+ words: Vec<Vec<char>>,+ word_set: HashSet<Vec<char>>,+}++impl Dictionary {+ pub fn create_from_static() -> Self {+ let english_words_raw = include_str!("../../../english_words.txt").replace('\r', "");++ Self {+ word_set: english_words_raw+ .split('\n')+ .filter(|word| !word.is_empty())+ .map(|word| word.chars().collect())+ .collect(),+ words: english_words_raw+ .split('\n')+ .filter(|word| !word.is_empty())+ .map(|word| word.chars().collect())+ .collect(),+ }+ }++ pub fn words_iter(&self) -> impl Iterator<Item = &'_ [char]> {+ self.words.iter().map(|v| v.as_slice())+ }++ pub fn contains_word(&self, word: &[char]) -> bool {+ self.word_set.contains(word)+ }+}diff --git a/lt-core/src/spell/mod.rs b/lt-core/src/spell/mod.rsindex 64ec94d7e..ecdb7fe99 100644--- a/lt-core/src/spell/mod.rs+++ b/lt-core/src/spell/mod.rs@@ -1,23 +1,23 @@-use crate::words::english_words;+pub use self::dictionary::Dictionary;++mod dictionary;/// Suggest a correct spelling for a given misspelled word./// [misspelled_word] is assumed to be quite small (n < 100)/// [max_edit_dist] relates to an optimization that allows the search algorithm to prune large portions of the search.-pub fn suggest_correct_spelling(+pub fn suggest_correct_spelling<'a>(misspelled_word: &[char],result_limit: usize,max_edit_dist: u8,-) -> Vec<&[char]> {- let words = english_words();-+ dictionary: &'a Dictionary,+) -> Vec<&'a [char]> {// 53 is the length of the longest word.let mut buf_a = Vec::with_capacity(53);let mut buf_b = Vec::with_capacity(53);- let pruned_words = words- .iter()+ let pruned_words = dictionary+ .words_iter().filter(|word| word.len().abs_diff(misspelled_word.len()) <= max_edit_dist as usize)- .cloned().filter_map(|word| {let dist = edit_distance_min_alloc(misspelled_word, word, &mut buf_a, &mut buf_b);@@ -51,10 +51,11 @@ pub fn suggest_correct_spelling_str(misspelled_word: impl AsRef<str>,result_limit: usize,max_edit_dist: u8,+ dictionary: &Dictionary,) -> Vec<String> {let chars: Vec<char> = misspelled_word.as_ref().chars().collect();- suggest_correct_spelling(&chars, result_limit, max_edit_dist)+ suggest_correct_spelling(&chars, result_limit, max_edit_dist, dictionary).into_iter().map(|word| word.iter().collect()).collect()diff --git a/lt-core/src/words.rs b/lt-core/src/words.rsdeleted file mode 100644index d0923126a..000000000--- a/lt-core/src/words.rs+++ /dev/null@@ -1,5 +0,0 @@-include!(concat!(env!("OUT_DIR"), "/english.rs"));--pub fn english_words() -> &'static [&'static [char]] {- &ENGLISH_WORDS-}diff --git a/lt-serve/src/main.rs b/lt-serve/src/main.rsindex c3ca8bb3c..478cfe142 100644--- a/lt-serve/src/main.rs+++ b/lt-serve/src/main.rs@@ -1,4 +1,4 @@-use lt_core::{lex_to_end, suggest_correct_spelling_str, Token};+use lt_core::{lex_to_end, suggest_correct_spelling_str, Dictionary, Token};use std::net::SocketAddr;use tracing::info;@@ -93,7 +93,9 @@ async fn spellcheck() -> (StatusCode, Json<SpellcheckResponse>) {info!("Spellcheck request for {:?}", payload.word);- let results = suggest_correct_spelling_str(payload.word, 5, 3);+ let dictionary = Dictionary::create_from_static();++ let results = suggest_correct_spelling_str(payload.word, 5, 3, &dictionary);(StatusCode::ACCEPTED,