Skip to content

Offline, privacy-first grammar checker. Fast, open-source, Rust-powered

feat: store just one curated dictionary regardless of thread count

Elijah Potter55d0c01469ad
diff --git a/Cargo.lock b/Cargo.lock
index f257f7759..1f837793e 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -28,9 +28,9 @@ dependencies = [
[[package]]
name = "allocator-api2"
-version = "0.2.18"
+version = "0.2.20"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5c6cb57a04249c6480766f7f7cef5467412af1490f8d1e243141daddada3264f"
+checksum = "45862d1c77f2228b9e10bc609d5bc203d86ebc9b87ad8d5d5167a6c9abf739d9"
[[package]]
name = "anes"
@@ -40,9 +40,9 @@ checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299"
[[package]]
name = "anstream"
-version = "0.6.17"
+version = "0.6.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "23a1e53f0f5d86382dafe1cf314783b2044280f406e7e1506368220ad11b1338"
+checksum = "8acc5369981196006228e28809f761875c0327210a891e941f4c683b3a99529b"
dependencies = [
"anstyle",
"anstyle-parse",
@@ -89,9 +89,9 @@ dependencies = [
[[package]]
name = "anyhow"
-version = "1.0.92"
+version = "1.0.93"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "74f37166d7d48a0284b99dd824694c26119c700b53bf0d1540cdb147dbdaaf13"
+checksum = "4c95c10ba0b00a02636238b814946408b1322d5ac4760326e6fb8ec956d85775"
[[package]]
name = "ariadne"
@@ -234,9 +234,9 @@ dependencies = [
[[package]]
name = "clap"
-version = "4.5.20"
+version = "4.5.21"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b97f376d85a664d5837dbae44bf546e6477a679ff6610010f17276f686d867e8"
+checksum = "fb3b4b9e5a7c7514dfa52869339ee98b3156b0bfb4e8a77c4ff4babb64b1604f"
dependencies = [
"clap_builder",
"clap_derive",
@@ -244,9 +244,9 @@ dependencies = [
[[package]]
name = "clap_builder"
-version = "4.5.20"
+version = "4.5.21"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "19bc80abd44e4bed93ca373a0704ccbd1b710dc5749406201bb018272808dc54"
+checksum = "b17a95aa67cc7b5ebd32aa5370189aa0d79069ef1c64ce893bd30fb24bff20ec"
dependencies = [
"anstream",
"anstyle",
@@ -268,9 +268,9 @@ dependencies = [
[[package]]
name = "clap_lex"
-version = "0.7.2"
+version = "0.7.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1462739cb27611015575c0c11df5df7601141071f07518d56fcc1be504cbec97"
+checksum = "afb84c814227b90d6895e01398aee0d8033c00e7466aca416fb6a8e0eb19d8a7"
[[package]]
name = "colorchoice"
@@ -598,9 +598,10 @@ dependencies = [
"blanket",
"criterion",
"fst",
- "hashbrown 0.15.0",
+ "hashbrown 0.15.1",
"is-macro",
"itertools 0.13.0",
+ "lazy_static",
"levenshtein_automata",
"ordered-float",
"paste",
@@ -677,9 +678,9 @@ checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1"
[[package]]
name = "hashbrown"
-version = "0.15.0"
+version = "0.15.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1e087f84d4f86bf4b218b927129862374b72199ae7d8657835f1e89000eea4fb"
+checksum = "3a9bfc1af68b1726ea47d3d5109de126281def866b33970e10fbab11b5dafab3"
dependencies = [
"allocator-api2",
"equivalent",
@@ -948,9 +949,9 @@ dependencies = [
[[package]]
name = "libc"
-version = "0.2.161"
+version = "0.2.164"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8e9489c2807c139ffd9c1794f4af0ebe86a828db53ecdc7fea2111d0fed085d1"
+checksum = "433bfe06b8c75da9b2e3fbea6e5329ff87748f0b144ef75306e674c3f6f7c13f"
[[package]]
name = "libredox"
@@ -1066,9 +1067,9 @@ checksum = "b410bbe7e14ab526a0e86877eb47c6996a2bd7746f027ba551028c925390e4e9"
[[package]]
name = "open"
-version = "5.3.0"
+version = "5.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "61a877bf6abd716642a53ef1b89fb498923a4afca5c754f9050b4d081c05c4b3"
+checksum = "3ecd52f0b8d15c40ce4820aa251ed5de032e5d91fab27f7db2f40d42a8bdf69c"
dependencies = [
"is-wsl",
"libc",
@@ -1301,9 +1302,9 @@ dependencies = [
[[package]]
name = "regex-automata"
-version = "0.4.8"
+version = "0.4.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "368758f23274712b504848e9d5a6f010445cc8b87a7cdb4d7cbee666c1288da3"
+checksum = "809e8dc61f6de73b46c85f4c96486310fe304c434cfa43669d7b40f711150908"
dependencies = [
"aho-corasick",
"memchr",
@@ -1354,9 +1355,9 @@ checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49"
[[package]]
name = "serde"
-version = "1.0.214"
+version = "1.0.215"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f55c3193aca71c12ad7890f1785d2b73e1b9f63a0bbc353c08ef26fe03fc56b5"
+checksum = "6513c1ad0b11a9376da888e3e0baa0077f1aed55c17f50e7b2397136129fb88f"
dependencies = [
"serde_derive",
]
@@ -1374,9 +1375,9 @@ dependencies = [
[[package]]
name = "serde_derive"
-version = "1.0.214"
+version = "1.0.215"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "de523f781f095e28fa605cdce0f8307e451cc0fd14e2eb4cd2e98a355b147766"
+checksum = "ad1e866f866923f252f05c889987993144fb74e722403468a4ebd70c3cd756c0"
dependencies = [
"proc-macro2",
"quote",
@@ -1385,9 +1386,9 @@ dependencies = [
[[package]]
name = "serde_json"
-version = "1.0.132"
+version = "1.0.133"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d726bfaff4b320266d395898905d0eba0345aae23b54aee3a737e260fd46db03"
+checksum = "c7fceb2473b9166b2294ef05efcb65a3db80803f0b03ef86a5fc88a2b85ee377"
dependencies = [
"itoa",
"memchr",
@@ -1479,18 +1480,18 @@ dependencies = [
[[package]]
name = "thiserror"
-version = "1.0.67"
+version = "1.0.69"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3b3c6efbfc763e64eb85c11c25320f0737cb7364c4b6336db90aa9ebe27a0bbd"
+checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52"
dependencies = [
"thiserror-impl",
]
[[package]]
name = "thiserror-impl"
-version = "1.0.67"
+version = "1.0.69"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b607164372e89797d78b8e23a6d67d5d1038c1c65efd52e1389ef8b77caba2a6"
+checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1"
dependencies = [
"proc-macro2",
"quote",
@@ -1529,9 +1530,9 @@ dependencies = [
[[package]]
name = "tokio"
-version = "1.41.0"
+version = "1.41.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "145f3413504347a2be84393cc8a7d2fb4d863b375909ea59f2158261aa258bbb"
+checksum = "22cfb5bee7a6a52939ca9224d6ac897bb669134078daa8735560897f69de4d33"
dependencies = [
"backtrace",
"bytes",
diff --git a/harper-core/Cargo.toml b/harper-core/Cargo.toml
index f90d413a3..900c39fb7 100644
--- a/harper-core/Cargo.toml
+++ b/harper-core/Cargo.toml
@@ -13,6 +13,7 @@ fst = "0.4.7"
hashbrown = { version = "0.15.0", features = ["serde"] }
is-macro = "0.3.6"
itertools = "0.13.0"
+lazy_static = "1.5.0"
ordered-float = { version = "4.5.0", features = ["serde"] }
paste = "1.0.14"
pulldown-cmark = "0.12.2"
diff --git a/harper-core/src/spell/fst_dictionary.rs b/harper-core/src/spell/fst_dictionary.rs
index 482919491..16d0bb1d9 100644
--- a/harper-core/src/spell/fst_dictionary.rs
+++ b/harper-core/src/spell/fst_dictionary.rs
@@ -5,6 +5,7 @@ use super::{
use fst::{map::StreamWithState, IntoStreamer, Map as FstMap, Streamer};
use hashbrown::HashMap;
use itertools::Itertools;
+use lazy_static::lazy_static;
use levenshtein_automata::{LevenshteinAutomatonBuilder, DFA};
use std::{cell::RefCell, sync::Arc};
@@ -37,9 +38,12 @@ fn uncached_inner_new() -> Arc<FstDictionary> {
const EXPECTED_DISTANCE: u8 = 3;
const TRANSPOSITION_COST_ONE: bool = false;
-thread_local! {
- static DICT: Arc<FstDictionary> = uncached_inner_new();
+lazy_static! {
+ static ref DICT: Arc<FstDictionary> = uncached_inner_new();
+}
+
+thread_local! {
// Builders are computationally expensive and do not depend on the word, so we store a
// collection of builders and the associated edit distance here.
// Currently, the edit distance we use is 3, but a value that does not exist in this
@@ -60,7 +64,7 @@ impl FstDictionary {
/// Create a dictionary from the curated dictionary included
/// in the Harper binary.
pub fn curated() -> Arc<Self> {
- DICT.with(|v| v.clone())
+ (*DICT).clone()
}
pub fn new(new_words: HashMap<CharString, WordMetadata>) -> Self {
diff --git a/harper-core/src/spell/full_dictionary.rs b/harper-core/src/spell/full_dictionary.rs
index d598465fe..dc2d26972 100644
--- a/harper-core/src/spell/full_dictionary.rs
+++ b/harper-core/src/spell/full_dictionary.rs
@@ -5,6 +5,7 @@ use super::{
};
use hashbrown::HashMap;
use itertools::Itertools;
+use lazy_static::lazy_static;
use smallvec::{SmallVec, ToSmallVec};
use std::sync::Arc;
@@ -56,8 +57,8 @@ fn uncached_inner_new() -> Arc<FullDictionary> {
})
}
-thread_local! {
- static DICT: Arc<FullDictionary> = uncached_inner_new();
+lazy_static! {
+ static ref DICT: Arc<FullDictionary> = uncached_inner_new();
}
impl FullDictionary {
@@ -73,7 +74,7 @@ impl FullDictionary {
/// in the Harper binary.
/// Consider using [`super::FstDictionary::curated()`] instead, as it is more performant for spellchecking.
pub fn curated() -> Arc<Self> {
- DICT.with(|v| v.clone())
+ (*DICT).clone()
}
/// Appends words to the dictionary.