Offline, privacy-first grammar checker. Fast, open-source, Rust-powered
Substantially improved `Dictionary` performance
Elijah Potter57fbd8ed5e02
diff --git a/Cargo.lock b/Cargo.lockindex 9a7a6b8cc..48fa88f8b 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -30,6 +30,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "77c3a9648d43b9cd48db467b3f87fdd6e146bcc88ab0180006cef2179fe11d01"dependencies = ["cfg-if",+ "getrandom","once_cell","version_check","zerocopy",@@ -44,12 +45,6 @@ dependencies = ["memchr",]-[[package]]-name = "allocator-api2"-version = "0.2.16"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "0942ffc6dcaadf03badf6e6a2d0228460359d5e34b57ccdc720b7382dfbd5ec5"-[[package]]name = "anes"version = "0.1.6"@@ -70,7 +65,7 @@ checksum = "a66537f1bb974b254c98ed142ff995236e81b9d0fe4db0575f46612cb15eb0f9"dependencies = ["proc-macro2","quote",- "syn 2.0.47",+ "syn",][[package]]@@ -167,39 +162,6 @@ version = "1.5.0"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "a2bd12c1caf447e69cd4528f47f94d203fd2582878ecb9e9465484c4148a8223"-[[package]]-name = "cached"-version = "0.47.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "69b0116662497bc24e4b177c90eaf8870e39e2714c3fcfa296327a93f593fc21"-dependencies = [- "ahash",- "cached_proc_macro",- "cached_proc_macro_types",- "hashbrown",- "instant",- "once_cell",- "thiserror",-]--[[package]]-name = "cached_proc_macro"-version = "0.18.1"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "c878c71c2821aa2058722038a59a67583a4240524687c6028571c9b395ded61f"-dependencies = [- "darling",- "proc-macro2",- "quote",- "syn 1.0.109",-]--[[package]]-name = "cached_proc_macro_types"-version = "0.1.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "3a4f925191b4367301851c6d99b09890311d74b0d43f274c0b34c86d308a3663"-[[package]]name = "cast"version = "0.3.0"@@ -290,6 +252,7 @@ dependencies = ["once_cell","oorandom","plotters",+ "rayon","regex","serde","serde_derive",@@ -309,39 +272,29 @@ dependencies = [][[package]]-name = "darling"-version = "0.14.4"+name = "crossbeam-deque"+version = "0.8.5"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "7b750cb3417fd1b327431a470f388520309479ab0bf5e323505daf0290cd3850"+checksum = "613f8cc01fe9cf1a3eb3d7f488fd2fa8388403e97039e2f73692932e291a770d"dependencies = [- "darling_core",- "darling_macro",+ "crossbeam-epoch",+ "crossbeam-utils",][[package]]-name = "darling_core"-version = "0.14.4"+name = "crossbeam-epoch"+version = "0.9.18"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "109c1ca6e6b7f82cc233a97004ea8ed7ca123a9af07a8230878fcfda9b158bf0"+checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e"dependencies = [- "fnv",- "ident_case",- "proc-macro2",- "quote",- "strsim",- "syn 1.0.109",+ "crossbeam-utils",][[package]]-name = "darling_macro"-version = "0.14.4"+name = "crossbeam-utils"+version = "0.8.19"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "a4aab4dbc9f7611d8b55048a3a16d2d010c2c8334e46304b40ac1cc14bf3b48e"-dependencies = [- "darling_core",- "quote",- "syn 1.0.109",-]+checksum = "248e3bacc7dc6baa3b21e405ee045c3047101a49145e7e9eca583ab4c2ca5345"[[package]]name = "either"@@ -407,6 +360,17 @@ dependencies = ["pin-utils",]+[[package]]+name = "getrandom"+version = "0.2.12"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "190092ea657667030ac6a35e305e62fc4dd69fd98ac98631e5d3a2b1575a12b5"+dependencies = [+ "cfg-if",+ "libc",+ "wasi",+]+[[package]]name = "gimli"version = "0.28.0"@@ -419,16 +383,6 @@ version = "1.8.2"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "eabb4a44450da02c90444cf74558da904edde8fb4e9035a9a6a4e15445af0bd7"-[[package]]-name = "hashbrown"-version = "0.14.3"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "290f1a1d9242c78d09ce40a5e87e7554ee637af1351968159f4952f028f75604"-dependencies = [- "ahash",- "allocator-api2",-]-[[package]]name = "hermit-abi"version = "0.3.3"@@ -492,21 +446,6 @@ dependencies = ["want",]-[[package]]-name = "ident_case"-version = "1.0.1"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39"--[[package]]-name = "instant"-version = "0.1.12"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c"-dependencies = [- "cfg-if",-]-[[package]]name = "is-macro"version = "0.3.0"@@ -517,7 +456,7 @@ dependencies = ["pmutil","proc-macro2","quote",- "syn 2.0.47",+ "syn",][[package]]@@ -602,11 +541,13 @@ checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f"name = "lt-core"version = "0.1.0"dependencies = [- "cached",+ "ahash","criterion","is-macro","itertools 0.11.0",+ "once_cell","serde",+ "smallvec",][[package]]@@ -700,9 +641,9 @@ dependencies = [[[package]]name = "once_cell"-version = "1.18.0"+version = "1.19.0"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d"+checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92"[[package]]name = "oorandom"@@ -762,7 +703,7 @@ checksum = "4359fd9c9171ec6e8c62926d6faaf553a8dc3f64e1507e76da7911b4f6a04405"dependencies = ["proc-macro2","quote",- "syn 2.0.47",+ "syn",][[package]]@@ -813,7 +754,7 @@ checksum = "52a40bc70c2c58040d2d8b167ba9a5ff59fc9dab7ad44771cfde3dcfde7a09c6"dependencies = ["proc-macro2","quote",- "syn 2.0.47",+ "syn",][[package]]@@ -834,6 +775,26 @@ dependencies = ["proc-macro2",]+[[package]]+name = "rayon"+version = "1.8.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "9c27db03db7734835b3f53954b534c91069375ce6ccaa2e065441e07d9b6cdb1"+dependencies = [+ "either",+ "rayon-core",+]++[[package]]+name = "rayon-core"+version = "1.12.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "5ce3fb6ad83f861aac485e76e1985cd109d9a3713802152be56c3b1f0e0658ed"+dependencies = [+ "crossbeam-deque",+ "crossbeam-utils",+]+[[package]]name = "redox_syscall"version = "0.4.1"@@ -935,7 +896,7 @@ checksum = "a3385e45322e8f9931410f01b3031ec534c3947d0e94c18049af4d9f9907d4e0"dependencies = ["proc-macro2","quote",- "syn 2.0.47",+ "syn",][[package]]@@ -991,9 +952,9 @@ dependencies = [[[package]]name = "smallvec"-version = "1.11.1"+version = "1.12.0"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "942b4a808e05215192e39f4ab80813e599068285906cc91aa64f923db842bd5a"+checksum = "2593d31f82ead8df961d8bd23a64c2ccf2eb5dd34b0a34bfb4dd54011c72009e"[[package]]name = "socket2"@@ -1015,23 +976,6 @@ dependencies = ["windows-sys",]-[[package]]-name = "strsim"-version = "0.10.1"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "ccbca6f34534eb78dbee83f6b2c9442fea7113f43d9e80ea320f0972ae5dc08d"--[[package]]-name = "syn"-version = "1.0.109"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237"-dependencies = [- "proc-macro2",- "quote",- "unicode-ident",-]-[[package]]name = "syn"version = "2.0.47"@@ -1049,26 +993,6 @@ version = "0.1.2"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160"-[[package]]-name = "thiserror"-version = "1.0.56"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "d54378c645627613241d077a3a79db965db602882668f9136ac42af9ecb730ad"-dependencies = [- "thiserror-impl",-]--[[package]]-name = "thiserror-impl"-version = "1.0.56"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "fa0faa943b50f3db30a20aa7e265dbc66076993efed8463e8de414e5d06d3471"-dependencies = [- "proc-macro2",- "quote",- "syn 2.0.47",-]-[[package]]name = "thread_local"version = "1.1.7"@@ -1116,7 +1040,7 @@ checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e"dependencies = ["proc-macro2","quote",- "syn 2.0.47",+ "syn",][[package]]@@ -1167,7 +1091,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7"dependencies = ["proc-macro2","quote",- "syn 2.0.47",+ "syn",][[package]]@@ -1275,7 +1199,7 @@ dependencies = ["once_cell","proc-macro2","quote",- "syn 2.0.47",+ "syn","wasm-bindgen-shared",]@@ -1297,7 +1221,7 @@ checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b"dependencies = ["proc-macro2","quote",- "syn 2.0.47",+ "syn","wasm-bindgen-backend","wasm-bindgen-shared",]@@ -1432,5 +1356,5 @@ checksum = "9ce1b18ccd8e73a9321186f97e46f9f04b778851177567b1975109d26a08d2a6"dependencies = ["proc-macro2","quote",- "syn 2.0.47",+ "syn",]diff --git a/lt-core/Cargo.toml b/lt-core/Cargo.tomlindex 26dd32c62..38a658447 100644--- a/lt-core/Cargo.toml+++ b/lt-core/Cargo.toml@@ -4,13 +4,15 @@ version = "0.1.0"edition = "2021"[dependencies]-cached = "0.47.0"+ahash = "0.8.7"is-macro = "0.3.0"itertools = "0.11.0"+once_cell = "1.19.0"serde = { version = "1.0.190", features = ["derive"] }+smallvec = "1.12.0"[dev-dependencies]-criterion = { version = "0.5.1", features = ["html_reports", "plotters", "cargo_bench_support"], default_features = false }+criterion = { version = "0.5.1" }[[bench]]name = "spellcheck"diff --git a/lt-core/benches/spellcheck.rs b/lt-core/benches/spellcheck.rsindex 1705893a1..efa6165f5 100644--- a/lt-core/benches/spellcheck.rs+++ b/lt-core/benches/spellcheck.rs@@ -7,10 +7,10 @@ fn spellcheck(dictionary: &Dictionary) {fn criterion_benchmark(c: &mut Criterion) {let mut group = c.benchmark_group("spellcheck");- group.sample_size(10000);let dictionary = Dictionary::new();+ group.bench_function("dict create", |b| b.iter(Dictionary::new));group.bench_function("hello 5", |b| b.iter(|| spellcheck(&dictionary)));}diff --git a/lt-core/src/linting/lint.rs b/lt-core/src/linting/lint.rsindex c90eb062b..3df4b96d7 100644--- a/lt-core/src/linting/lint.rs+++ b/lt-core/src/linting/lint.rs@@ -1,6 +1,6 @@use serde::{Deserialize, Serialize};-use crate::{document::Document, span::Span};+use crate::{document::Document, span::Span, Dictionary};#[derive(Debug, Clone, Serialize, Deserialize)]pub struct Lint {@@ -23,4 +23,4 @@ pub enum Suggestion {ReplaceWith(Vec<char>),}-pub type Linter = fn(document: &Document) -> Vec<Lint>;+pub type Linter = fn(document: &Document, dictionary: &Dictionary) -> Vec<Lint>;diff --git a/lt-core/src/linting/mod.rs b/lt-core/src/linting/mod.rsindex 268dd59f3..70147b38f 100644--- a/lt-core/src/linting/mod.rs+++ b/lt-core/src/linting/mod.rs@@ -6,11 +6,11 @@ mod wrong_quotes;pub use lint::{Lint, LintKind, Suggestion};-use crate::Document;+use crate::{Dictionary, Document};use self::lint::Linter;-pub fn all_linters(document: &Document) -> Vec<Lint> {+pub fn all_linters(document: &Document, dictionary: &Dictionary) -> Vec<Lint> {let mut lints = Vec::new();let linters: [Linter; 4] = [@@ -21,7 +21,7 @@ pub fn all_linters(document: &Document) -> Vec<Lint> {];for linter in linters {- lints.append(&mut linter(document));+ lints.append(&mut linter(document, dictionary));}lintsdiff --git a/lt-core/src/linting/sentence_capitalization.rs b/lt-core/src/linting/sentence_capitalization.rsindex 8479c12b1..b521759af 100644--- a/lt-core/src/linting/sentence_capitalization.rs+++ b/lt-core/src/linting/sentence_capitalization.rs@@ -1,11 +1,11 @@use itertools::Itertools;-use crate::{document::Document, parsing::TokenStringExt, Lint, LintKind};+use crate::{document::Document, parsing::TokenStringExt, Dictionary, Lint, LintKind};use super::lint::Suggestion;/// A linter that checks to make sure the first word of each sentence is capitalized.-pub fn sentence_capitalization_lint(document: &Document) -> Vec<Lint> {+pub fn sentence_capitalization_lint(document: &Document, _dictionary: &Dictionary) -> Vec<Lint> {let mut lints = Vec::new();for sentence in document.sentences() {diff --git a/lt-core/src/linting/spell_check.rs b/lt-core/src/linting/spell_check.rsindex 173f46b35..b028d4878 100644--- a/lt-core/src/linting/spell_check.rs+++ b/lt-core/src/linting/spell_check.rs@@ -2,7 +2,7 @@ use crate::{document::Document, suggest_correct_spelling, Dictionary, Lint, Lintuse super::lint::Suggestion;-pub fn spell_check(document: &Document) -> Vec<Lint> {+pub fn spell_check(document: &Document, _dictionary: &Dictionary) -> Vec<Lint> {let mut lints = Vec::new();let dictionary = Dictionary::new();diff --git a/lt-core/src/linting/unclosed_quotes.rs b/lt-core/src/linting/unclosed_quotes.rsindex 3276c9132..9bffe7c7b 100644--- a/lt-core/src/linting/unclosed_quotes.rs+++ b/lt-core/src/linting/unclosed_quotes.rs@@ -1,6 +1,8 @@-use crate::{document::Document, parsing::Quote, Lint, LintKind, Punctuation, TokenKind};+use crate::{+ document::Document, parsing::Quote, Dictionary, Lint, LintKind, Punctuation, TokenKind,+};-pub fn unclosed_quotes(document: &Document) -> Vec<Lint> {+pub fn unclosed_quotes(document: &Document, _dictionary: &Dictionary) -> Vec<Lint> {let mut lints = Vec::new();// TODO: Try zipping quote positionsdiff --git a/lt-core/src/linting/wrong_quotes.rs b/lt-core/src/linting/wrong_quotes.rsindex 80bb9027e..8a29174bd 100644--- a/lt-core/src/linting/wrong_quotes.rs+++ b/lt-core/src/linting/wrong_quotes.rs@@ -1,8 +1,9 @@use crate::{- document::Document, parsing::Quote, Lint, LintKind, Punctuation, Suggestion, Token, TokenKind,+ document::Document, parsing::Quote, Dictionary, Lint, LintKind, Punctuation, Suggestion, Token,+ TokenKind,};-pub fn wrong_quotes(document: &Document) -> Vec<Lint> {+pub fn wrong_quotes(document: &Document, _dictionary: &Dictionary) -> Vec<Lint> {document.iter_quote_indices().zip(document.iter_quotes())diff --git a/lt-core/src/spell/dictionary.rs b/lt-core/src/spell/dictionary.rsindex 8db100edd..f59c5e328 100644--- a/lt-core/src/spell/dictionary.rs+++ b/lt-core/src/spell/dictionary.rs@@ -1,6 +1,10 @@-use std::{collections::HashSet, iter};+use std::{borrow::Borrow, hash::Hasher};-use cached::proc_macro::cached;+use ahash::{AHashSet, AHasher};+use once_cell::sync::Lazy;+use smallvec::SmallVec;++type DictWord = SmallVec<[char; 16]>;#[derive(Debug, Clone)]pub struct Dictionary {@@ -10,19 +14,18 @@ pub struct Dictionary {/// This is likely due to increased locality :shrug:.////// This list is sorted by word length (i.e. the shortest words are first).- words: Vec<Vec<char>>,+ words: Vec<DictWord>,/// A lookup list for each word length.- /// Each index of this list will return the first index of [`Self::words`] that has a word of- /// that length.+ /// Each index of this list will return the first index of [`Self::words`] that has a word+ /// whose index is that length.word_len_starts: Vec<usize>,- word_set: HashSet<Vec<char>>,+ word_set: AHashSet<u64>,}-#[cached]-fn cached_inner_new() -> Dictionary {+fn uncached_inner_new() -> Dictionary {let english_words_raw = include_str!("../../../english_words.txt").replace('\r', "");- let mut words: Vec<Vec<char>> = english_words_raw+ let mut words: Vec<DictWord> = english_words_raw.split('\n').filter(|word| !word.is_empty()).map(|word| word.chars().collect())@@ -32,28 +35,38 @@ fn cached_inner_new() -> Dictionary {let mut word_len_starts = vec![0, 0];- for (index, len) in words.iter().map(Vec::len).enumerate() {+ for (index, len) in words.iter().map(SmallVec::len).enumerate() {if word_len_starts.len() == len {word_len_starts.push(index);}}Dictionary {- word_set: HashSet::from_iter(words.iter().cloned()),+ word_set: AHashSet::from_iter(words.iter().map(|v| hash_word(v.as_slice()))),word_len_starts,words,}}+fn hash_word(word: &[char]) -> u64 {+ let mut hasher = AHasher::default();+ for c in word {+ hasher.write_u32(*c as u32);+ }+ hasher.finish()+}++static DICT: Lazy<Dictionary> = Lazy::new(uncached_inner_new);+impl Dictionary {- pub fn new() -> Self {- cached_inner_new()+ pub fn new() -> &'static Self {+ &DICT}/// Iterate over all the words in the dicitonary of a given lengthpub fn words_with_len_iter(&self, len: usize) -> Box<dyn Iterator<Item = &'_ [char]> + '_> {if len == 0 || len >= self.word_len_starts.len() {- return Box::new(iter::empty());+ return Box::new(std::iter::empty());}let start = self.word_len_starts[len];@@ -73,12 +86,6 @@ impl Dictionary {pub fn contains_word(&self, word: &[char]) -> bool {let lowercase: Vec<_> = word.iter().flat_map(|c| c.to_lowercase()).collect();- self.word_set.contains(word) || self.word_set.contains(&lowercase)- }-}--impl Default for Dictionary {- fn default() -> Self {- Self::new()+ self.word_set.contains(&hash_word(word)) || self.word_set.contains(&hash_word(&lowercase))}}diff --git a/lt-serve/src/main.rs b/lt-serve/src/main.rsindex 019386149..653f32dca 100644--- a/lt-serve/src/main.rs+++ b/lt-serve/src/main.rs@@ -1,6 +1,6 @@#![allow(dead_code)]-use lt_core::{all_linters, Document, FatToken, Lint, Span, Suggestion};+use lt_core::{all_linters, Dictionary, Document, FatToken, Lint, Span, Suggestion};use std::net::SocketAddr;use tokio::time::Instant;use tracing::{debug, info, Level};@@ -88,9 +88,10 @@ struct ParseResponse {async fn lint(Json(payload): Json<LintRequest>) -> (StatusCode, Json<LintResponse>) {let text = payload.text;+ let dictionary = Dictionary::new();let document = Document::new(&text);- let lints = all_linters(&document);+ let lints = all_linters(&document, dictionary);(StatusCode::ACCEPTED, Json(LintResponse { lints }))}