Skip to content

Offline, privacy-first grammar checker. Fast, open-source, Rust-powered

Switched away from build script - it was just too complex

Elijah Potter0a397bdec599
diff --git a/Cargo.lock b/Cargo.lock
index f68ffc48a..773e233d9 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -518,7 +518,6 @@ version = "0.1.0"
dependencies = [
"criterion",
"is-macro",
- "regex",
"serde",
]
diff --git a/lt-core/Cargo.toml b/lt-core/Cargo.toml
index 2c03d394b..7dd697de6 100644
--- a/lt-core/Cargo.toml
+++ b/lt-core/Cargo.toml
@@ -2,15 +2,11 @@
name = "lt-core"
version = "0.1.0"
edition = "2021"
-build = "build.rs"
[dependencies]
is-macro = "0.3.0"
serde = { version = "1.0.189", features = ["derive"] }
-[build-dependencies]
-regex = "1.10.2"
-
[dev-dependencies]
criterion = { version = "0.5.1", features = ["html_reports"] }
diff --git a/lt-core/benches/spellcheck.rs b/lt-core/benches/spellcheck.rs
index af6a29dc7..3eae41552 100644
--- a/lt-core/benches/spellcheck.rs
+++ b/lt-core/benches/spellcheck.rs
@@ -1,12 +1,14 @@
use criterion::{black_box, criterion_group, criterion_main, Criterion};
-use lt_core::suggest_correct_spelling_str;
+use lt_core::{suggest_correct_spelling_str, Dictionary};
-fn spellcheck() {
- suggest_correct_spelling_str("hello", 5, 3);
+fn spellcheck(dictionary: &Dictionary) {
+ suggest_correct_spelling_str("hello", 5, 3, dictionary);
}
fn criterion_benchmark(c: &mut Criterion) {
- c.bench_function("hello 5", |b| b.iter(|| spellcheck()));
+ let dictionary = Dictionary::create_from_static();
+
+ c.bench_function("hello 5", |b| b.iter(|| spellcheck(&dictionary)));
}
criterion_group!(benches, criterion_benchmark);
diff --git a/lt-core/build.rs b/lt-core/build.rs
deleted file mode 100644
index 39b2c5d24..000000000
--- a/lt-core/build.rs
+++ /dev/null
@@ -1,36 +0,0 @@
-use regex::Regex;
-use std::io::Write;
-use std::{env, fs::File, io::BufWriter, path::Path};
-
-// Generate the code needed to embed the word list as a static array to be included in the binary.
-fn main() {
- let english_words_raw = include_str!("../english_words.txt").replace('\r', "");
-
- let re = Regex::new(r"[\r-']").unwrap();
- let cleaned = re.replace_all(&english_words_raw, "");
-
- let words: Vec<_> = cleaned
- .split('\n')
- .filter(|word| !word.is_empty())
- .collect();
-
- let path = Path::new(&env::var("OUT_DIR").unwrap()).join("english.rs");
- let mut out_file = BufWriter::new(File::create(path).unwrap());
-
- write!(
- out_file,
- "#[allow(clippy::all)] static ENGLISH_WORDS: [&[char]; {}] = [",
- words.len()
- )
- .unwrap();
-
- for word in words {
- write!(out_file, "&[").unwrap();
- for character in word.chars() {
- write!(out_file, "'{}', ", character).unwrap();
- }
- write!(out_file, "],").unwrap();
- }
-
- write!(out_file, "];").unwrap();
-}
diff --git a/lt-core/src/lib.rs b/lt-core/src/lib.rs
index cc2917ac3..3004e0757 100644
--- a/lt-core/src/lib.rs
+++ b/lt-core/src/lib.rs
@@ -3,9 +3,9 @@ mod lint;
mod span;
mod spell;
mod token;
-mod words;
pub use lexer::{lex_to_end, lex_to_end_str};
pub use lint::{Lint, LintKind};
+pub use spell::Dictionary;
pub use spell::{suggest_correct_spelling, suggest_correct_spelling_str};
pub use token::{Token, TokenKind};
diff --git a/lt-core/src/spell/dictionary.rs b/lt-core/src/spell/dictionary.rs
new file mode 100644
index 000000000..e43bc14a7
--- /dev/null
+++ b/lt-core/src/spell/dictionary.rs
@@ -0,0 +1,35 @@
+use std::collections::HashSet;
+
+pub struct Dictionary {
+ /// Storing a separate [Vec] for iterations speeds up spellchecking by ~16% at the cost of
+ /// additional memory.
+ words: Vec<Vec<char>>,
+ word_set: HashSet<Vec<char>>,
+}
+
+impl Dictionary {
+ pub fn create_from_static() -> Self {
+ let english_words_raw = include_str!("../../../english_words.txt").replace('\r', "");
+
+ Self {
+ word_set: english_words_raw
+ .split('\n')
+ .filter(|word| !word.is_empty())
+ .map(|word| word.chars().collect())
+ .collect(),
+ words: english_words_raw
+ .split('\n')
+ .filter(|word| !word.is_empty())
+ .map(|word| word.chars().collect())
+ .collect(),
+ }
+ }
+
+ pub fn words_iter(&self) -> impl Iterator<Item = &'_ [char]> {
+ self.words.iter().map(|v| v.as_slice())
+ }
+
+ pub fn contains_word(&self, word: &[char]) -> bool {
+ self.word_set.contains(word)
+ }
+}
diff --git a/lt-core/src/spell/mod.rs b/lt-core/src/spell/mod.rs
index 64ec94d7e..ecdb7fe99 100644
--- a/lt-core/src/spell/mod.rs
+++ b/lt-core/src/spell/mod.rs
@@ -1,23 +1,23 @@
-use crate::words::english_words;
+pub use self::dictionary::Dictionary;
+
+mod dictionary;
/// Suggest a correct spelling for a given misspelled word.
/// [misspelled_word] is assumed to be quite small (n < 100)
/// [max_edit_dist] relates to an optimization that allows the search algorithm to prune large portions of the search.
-pub fn suggest_correct_spelling(
+pub fn suggest_correct_spelling<'a>(
misspelled_word: &[char],
result_limit: usize,
max_edit_dist: u8,
-) -> Vec<&[char]> {
- let words = english_words();
-
+ dictionary: &'a Dictionary,
+) -> Vec<&'a [char]> {
// 53 is the length of the longest word.
let mut buf_a = Vec::with_capacity(53);
let mut buf_b = Vec::with_capacity(53);
- let pruned_words = words
- .iter()
+ let pruned_words = dictionary
+ .words_iter()
.filter(|word| word.len().abs_diff(misspelled_word.len()) <= max_edit_dist as usize)
- .cloned()
.filter_map(|word| {
let dist = edit_distance_min_alloc(misspelled_word, word, &mut buf_a, &mut buf_b);
@@ -51,10 +51,11 @@ pub fn suggest_correct_spelling_str(
misspelled_word: impl AsRef<str>,
result_limit: usize,
max_edit_dist: u8,
+ dictionary: &Dictionary,
) -> Vec<String> {
let chars: Vec<char> = misspelled_word.as_ref().chars().collect();
- suggest_correct_spelling(&chars, result_limit, max_edit_dist)
+ suggest_correct_spelling(&chars, result_limit, max_edit_dist, dictionary)
.into_iter()
.map(|word| word.iter().collect())
.collect()
diff --git a/lt-core/src/words.rs b/lt-core/src/words.rs
deleted file mode 100644
index d0923126a..000000000
--- a/lt-core/src/words.rs
+++ /dev/null
@@ -1,5 +0,0 @@
-include!(concat!(env!("OUT_DIR"), "/english.rs"));
-
-pub fn english_words() -> &'static [&'static [char]] {
- &ENGLISH_WORDS
-}
diff --git a/lt-serve/src/main.rs b/lt-serve/src/main.rs
index c3ca8bb3c..478cfe142 100644
--- a/lt-serve/src/main.rs
+++ b/lt-serve/src/main.rs
@@ -1,4 +1,4 @@
-use lt_core::{lex_to_end, suggest_correct_spelling_str, Token};
+use lt_core::{lex_to_end, suggest_correct_spelling_str, Dictionary, Token};
use std::net::SocketAddr;
use tracing::info;
@@ -93,7 +93,9 @@ async fn spellcheck(
) -> (StatusCode, Json<SpellcheckResponse>) {
info!("Spellcheck request for {:?}", payload.word);
- let results = suggest_correct_spelling_str(payload.word, 5, 3);
+ let dictionary = Dictionary::create_from_static();
+
+ let results = suggest_correct_spelling_str(payload.word, 5, 3, &dictionary);
(
StatusCode::ACCEPTED,