Skip to content

Offline, privacy-first grammar checker. Fast, open-source, Rust-powered

Spellcheck is faster and capitalization check works more consistently

Elijah Potter5ee18a011ab0
diff --git a/Cargo.lock b/Cargo.lock
index 96f82d97e..9a7a6b8cc 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -23,6 +23,18 @@ version = "1.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe"
+[[package]]
+name = "ahash"
+version = "0.8.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "77c3a9648d43b9cd48db467b3f87fdd6e146bcc88ab0180006cef2179fe11d01"
+dependencies = [
+ "cfg-if",
+ "once_cell",
+ "version_check",
+ "zerocopy",
+]
+
[[package]]
name = "aho-corasick"
version = "1.1.2"
@@ -32,6 +44,12 @@ dependencies = [
"memchr",
]
+[[package]]
+name = "allocator-api2"
+version = "0.2.16"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0942ffc6dcaadf03badf6e6a2d0228460359d5e34b57ccdc720b7382dfbd5ec5"
+
[[package]]
name = "anes"
version = "0.1.6"
@@ -52,7 +70,7 @@ checksum = "a66537f1bb974b254c98ed142ff995236e81b9d0fe4db0575f46612cb15eb0f9"
dependencies = [
"proc-macro2",
"quote",
- "syn",
+ "syn 2.0.47",
]
[[package]]
@@ -149,6 +167,39 @@ version = "1.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a2bd12c1caf447e69cd4528f47f94d203fd2582878ecb9e9465484c4148a8223"
+[[package]]
+name = "cached"
+version = "0.47.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "69b0116662497bc24e4b177c90eaf8870e39e2714c3fcfa296327a93f593fc21"
+dependencies = [
+ "ahash",
+ "cached_proc_macro",
+ "cached_proc_macro_types",
+ "hashbrown",
+ "instant",
+ "once_cell",
+ "thiserror",
+]
+
+[[package]]
+name = "cached_proc_macro"
+version = "0.18.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c878c71c2821aa2058722038a59a67583a4240524687c6028571c9b395ded61f"
+dependencies = [
+ "darling",
+ "proc-macro2",
+ "quote",
+ "syn 1.0.109",
+]
+
+[[package]]
+name = "cached_proc_macro_types"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3a4f925191b4367301851c6d99b09890311d74b0d43f274c0b34c86d308a3663"
+
[[package]]
name = "cast"
version = "0.3.0"
@@ -257,6 +308,41 @@ dependencies = [
"itertools 0.10.5",
]
+[[package]]
+name = "darling"
+version = "0.14.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7b750cb3417fd1b327431a470f388520309479ab0bf5e323505daf0290cd3850"
+dependencies = [
+ "darling_core",
+ "darling_macro",
+]
+
+[[package]]
+name = "darling_core"
+version = "0.14.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "109c1ca6e6b7f82cc233a97004ea8ed7ca123a9af07a8230878fcfda9b158bf0"
+dependencies = [
+ "fnv",
+ "ident_case",
+ "proc-macro2",
+ "quote",
+ "strsim",
+ "syn 1.0.109",
+]
+
+[[package]]
+name = "darling_macro"
+version = "0.14.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a4aab4dbc9f7611d8b55048a3a16d2d010c2c8334e46304b40ac1cc14bf3b48e"
+dependencies = [
+ "darling_core",
+ "quote",
+ "syn 1.0.109",
+]
+
[[package]]
name = "either"
version = "1.9.0"
@@ -333,6 +419,16 @@ version = "1.8.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "eabb4a44450da02c90444cf74558da904edde8fb4e9035a9a6a4e15445af0bd7"
+[[package]]
+name = "hashbrown"
+version = "0.14.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "290f1a1d9242c78d09ce40a5e87e7554ee637af1351968159f4952f028f75604"
+dependencies = [
+ "ahash",
+ "allocator-api2",
+]
+
[[package]]
name = "hermit-abi"
version = "0.3.3"
@@ -396,6 +492,21 @@ dependencies = [
"want",
]
+[[package]]
+name = "ident_case"
+version = "1.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39"
+
+[[package]]
+name = "instant"
+version = "0.1.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c"
+dependencies = [
+ "cfg-if",
+]
+
[[package]]
name = "is-macro"
version = "0.3.0"
@@ -406,7 +517,7 @@ dependencies = [
"pmutil",
"proc-macro2",
"quote",
- "syn",
+ "syn 2.0.47",
]
[[package]]
@@ -491,6 +602,7 @@ checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f"
name = "lt-core"
version = "0.1.0"
dependencies = [
+ "cached",
"criterion",
"is-macro",
"itertools 0.11.0",
@@ -650,7 +762,7 @@ checksum = "4359fd9c9171ec6e8c62926d6faaf553a8dc3f64e1507e76da7911b4f6a04405"
dependencies = [
"proc-macro2",
"quote",
- "syn",
+ "syn 2.0.47",
]
[[package]]
@@ -701,7 +813,7 @@ checksum = "52a40bc70c2c58040d2d8b167ba9a5ff59fc9dab7ad44771cfde3dcfde7a09c6"
dependencies = [
"proc-macro2",
"quote",
- "syn",
+ "syn 2.0.47",
]
[[package]]
@@ -823,7 +935,7 @@ checksum = "a3385e45322e8f9931410f01b3031ec534c3947d0e94c18049af4d9f9907d4e0"
dependencies = [
"proc-macro2",
"quote",
- "syn",
+ "syn 2.0.47",
]
[[package]]
@@ -903,6 +1015,23 @@ dependencies = [
"windows-sys",
]
+[[package]]
+name = "strsim"
+version = "0.10.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ccbca6f34534eb78dbee83f6b2c9442fea7113f43d9e80ea320f0972ae5dc08d"
+
+[[package]]
+name = "syn"
+version = "1.0.109"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "unicode-ident",
+]
+
[[package]]
name = "syn"
version = "2.0.47"
@@ -920,6 +1049,26 @@ version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160"
+[[package]]
+name = "thiserror"
+version = "1.0.56"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d54378c645627613241d077a3a79db965db602882668f9136ac42af9ecb730ad"
+dependencies = [
+ "thiserror-impl",
+]
+
+[[package]]
+name = "thiserror-impl"
+version = "1.0.56"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fa0faa943b50f3db30a20aa7e265dbc66076993efed8463e8de414e5d06d3471"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.47",
+]
+
[[package]]
name = "thread_local"
version = "1.1.7"
@@ -967,7 +1116,7 @@ checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e"
dependencies = [
"proc-macro2",
"quote",
- "syn",
+ "syn 2.0.47",
]
[[package]]
@@ -1018,7 +1167,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7"
dependencies = [
"proc-macro2",
"quote",
- "syn",
+ "syn 2.0.47",
]
[[package]]
@@ -1074,6 +1223,12 @@ version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d"
+[[package]]
+name = "version_check"
+version = "0.9.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f"
+
[[package]]
name = "walkdir"
version = "2.4.0"
@@ -1120,7 +1275,7 @@ dependencies = [
"once_cell",
"proc-macro2",
"quote",
- "syn",
+ "syn 2.0.47",
"wasm-bindgen-shared",
]
@@ -1142,7 +1297,7 @@ checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b"
dependencies = [
"proc-macro2",
"quote",
- "syn",
+ "syn 2.0.47",
"wasm-bindgen-backend",
"wasm-bindgen-shared",
]
@@ -1259,3 +1414,23 @@ name = "windows_x86_64_msvc"
version = "0.48.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538"
+
+[[package]]
+name = "zerocopy"
+version = "0.7.32"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "74d4d3961e53fa4c9a25a8637fc2bfaf2595b3d3ae34875568a5cf64787716be"
+dependencies = [
+ "zerocopy-derive",
+]
+
+[[package]]
+name = "zerocopy-derive"
+version = "0.7.32"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9ce1b18ccd8e73a9321186f97e46f9f04b778851177567b1975109d26a08d2a6"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.47",
+]
diff --git a/alice.txt b/alice.txt
index cce353fb4..36a55fdac 100644
--- a/alice.txt
+++ b/alice.txt
@@ -10,7 +10,7 @@ sir---" The Rabbit started violently, dropped the white kid gloves and
the fan, and scurried away into the darkness as hard as he could go.
Alice took up the fan and gloves, and, as the hall was very hot, she
-kept fanning herself all the time she went on talking! "Dear, dear! How
+kept fanning hrself all the time she went on talking! "Dear, dear! How
queer everything is today! And yesterday things went on just as usual.
I wonder if I've been changed during the night? Let me think: was I
the same when I got up this morning? I almost think I can remember
diff --git a/english_words.txt b/english_words.txt
index 82fb0a0c9..e115bcd95 100644
--- a/english_words.txt
+++ b/english_words.txt
@@ -1,6 +1,10 @@
+that's
+i've
+I'm
+that's
+won't
+i
a
-aa
-aaa
aah
aahed
aahing
@@ -4892,7 +4896,6 @@ aequor
aequoreal
aequorin
aequorins
-aer
aerage
aeraria
aerarian
@@ -25415,7 +25418,6 @@ azurite
azurites
azurmalachite
azurous
-b
ba
baa
baaed
@@ -43828,7 +43830,6 @@ bwanas
bx
bxs
bz
-c
ca
caaba
caam
@@ -75935,7 +75936,6 @@ czechoslovakians
czechoslovaks
czechs
czigany
-d
da
daalder
dab
@@ -94668,7 +94668,6 @@ dzeron
dziggetai
dzo
dzungar
-e
ea
eably
eaceworm
@@ -108865,7 +108864,6 @@ ezba
ezekiel
ezod
ezra
-f
fa
faade
faailk
@@ -120758,7 +120756,6 @@ fw
fwd
fwelling
fz
-g
ga
gaatch
gab
@@ -131711,7 +131708,6 @@ gwerziou
gwine
gwiniad
gwyniad
-h
ha
haab
haaf
@@ -145454,8 +145450,6 @@ hwan
hwy
hwyl
hwt
-i
-y
ia
ya
yaba
@@ -159796,7 +159790,6 @@ izzard
izzards
izzat
izzy
-j
ja
jaalin
jaap
@@ -162636,7 +162629,6 @@ juxtaterrestrial
juxtatropical
juza
jwahar
-k
ka
kaaba
kaama
@@ -166588,7 +166580,6 @@ kwazoku
kwela
kwhr
kwintra
-l
la
laager
laagered
@@ -176590,7 +176581,6 @@ lwop
lwp
lx
lxx
-m
ma
maad
maam
@@ -196395,7 +196385,6 @@ mwalimu
mxd
mzee
mzungu
-n
na
naa
naam
@@ -209854,7 +209843,6 @@ nuzzlers
nuzzles
nuzzling
nv
-o
oad
oadal
oaf
@@ -222535,7 +222523,6 @@ ozophene
ozostomia
ozotype
ozs
-p
pa
paal
paaneleinrg
@@ -257395,7 +257382,6 @@ pvt
pwca
pwr
pwt
-q
qabbala
qabbalah
qadarite
@@ -259188,7 +259174,6 @@ qurti
qurush
qurushes
qv
-r
ra
raad
raadzaal
@@ -275971,7 +275956,6 @@ rvulsant
rwd
rwy
rwound
-s
sa
saa
saad
@@ -314735,7 +314719,6 @@ szaibelyite
szekler
szlachta
szopelka
-t
ta
taa
taal
@@ -333554,7 +333537,6 @@ tzontle
tzotzil
tzuris
tzutuhil
-u
uayeb
uakari
ualis
@@ -356321,7 +356303,6 @@ uzaron
uzbak
uzbeg
uzbek
-v
va
vaad
vaadim
@@ -361650,7 +361631,6 @@ vulvovaginitis
vum
vv
vvll
-w
wa
waac
waag
@@ -367085,7 +367065,6 @@ wonners
wonning
wonnot
wons
-wont
wonted
wontedly
wontedness
@@ -368209,7 +368188,6 @@ wuzzy
wuzzle
wuzzled
wuzzling
-x
xalostockite
xanthaline
xanthamic
@@ -368716,7 +368694,6 @@ xxiii
xxiv
xxv
xxx
-z
za
zabaean
zabaglione
diff --git a/lt-core/Cargo.toml b/lt-core/Cargo.toml
index 9db6269dc..26dd32c62 100644
--- a/lt-core/Cargo.toml
+++ b/lt-core/Cargo.toml
@@ -4,6 +4,7 @@ version = "0.1.0"
edition = "2021"
[dependencies]
+cached = "0.47.0"
is-macro = "0.3.0"
itertools = "0.11.0"
serde = { version = "1.0.190", features = ["derive"] }
diff --git a/lt-core/benches/spellcheck.rs b/lt-core/benches/spellcheck.rs
index ec82a82e9..1705893a1 100644
--- a/lt-core/benches/spellcheck.rs
+++ b/lt-core/benches/spellcheck.rs
@@ -6,9 +6,12 @@ fn spellcheck(dictionary: &Dictionary) {
}
fn criterion_benchmark(c: &mut Criterion) {
- let dictionary = Dictionary::create_from_static();
+ let mut group = c.benchmark_group("spellcheck");
+ group.sample_size(10000);
- c.bench_function("hello 5", |b| b.iter(|| spellcheck(&dictionary)));
+ let dictionary = Dictionary::new();
+
+ group.bench_function("hello 5", |b| b.iter(|| spellcheck(&dictionary)));
}
criterion_group!(benches, criterion_benchmark);
diff --git a/lt-core/src/linting/sentence_capitalization.rs b/lt-core/src/linting/sentence_capitalization.rs
index 232d56ef3..1e9186d35 100644
--- a/lt-core/src/linting/sentence_capitalization.rs
+++ b/lt-core/src/linting/sentence_capitalization.rs
@@ -1,6 +1,6 @@
use itertools::Itertools;
-use crate::{document::Document, Lint, LintKind};
+use crate::{document::Document, parsing::TokenStringExt, Lint, LintKind};
use super::lint::Suggestion;
@@ -9,7 +9,12 @@ pub fn sentence_capitalization_lint(document: &Document) -> Vec<Lint> {
let mut lints = Vec::new();
for sentence in document.sentences() {
- if let Some(first_word) = sentence.first() {
+ dbg!(document.get_content_string(crate::Span {
+ start: sentence.first().unwrap().span.start,
+ end: sentence.last().unwrap().span.start,
+ }));
+
+ if let Some(first_word) = sentence.first_word() {
let letters = document.get_span_content(first_word.span);
if let Some(first_letter) = letters.first() {
diff --git a/lt-core/src/linting/spell_check.rs b/lt-core/src/linting/spell_check.rs
index 8dcdea6ba..81b17174f 100644
--- a/lt-core/src/linting/spell_check.rs
+++ b/lt-core/src/linting/spell_check.rs
@@ -5,7 +5,7 @@ use super::lint::Suggestion;
pub fn spell_check(document: &Document) -> Vec<Lint> {
let mut lints = Vec::new();
- let dictionary = Dictionary::create_from_static();
+ let dictionary = Dictionary::new();
for word in document.words() {
let word_chars = document.get_span_content(word.span);
diff --git a/lt-core/src/parsing/lexer.rs b/lt-core/src/parsing/lexer.rs
index 705c06ee9..4fc4f1db7 100644
--- a/lt-core/src/parsing/lexer.rs
+++ b/lt-core/src/parsing/lexer.rs
@@ -172,8 +172,6 @@ lex_punctuation! {
"\"" => Quote,
"," => Comma,
"-" => Hyphen,
- "’" => Apostrophe,
- "'" => Apostrophe,
"[" => OpenSquare,
"]" => CloseSquare,
"(" => OpenRound,
diff --git a/lt-core/src/parsing/mod.rs b/lt-core/src/parsing/mod.rs
index 12ca6be01..4e91bcc1c 100644
--- a/lt-core/src/parsing/mod.rs
+++ b/lt-core/src/parsing/mod.rs
@@ -3,4 +3,4 @@ mod lexer;
mod token;
pub use lexer::{lex_to_end, lex_to_end_str};
-pub use token::{FatToken, Punctuation, Token, TokenKind};
+pub use token::{FatToken, Punctuation, Token, TokenKind, TokenStringExt};
diff --git a/lt-core/src/parsing/token.rs b/lt-core/src/parsing/token.rs
index 26ddca2dd..22e3c98db 100644
--- a/lt-core/src/parsing/token.rs
+++ b/lt-core/src/parsing/token.rs
@@ -59,8 +59,6 @@ pub enum Punctuation {
Comma,
/// -
Hyphen,
- /// ' or ’
- Apostrophe,
/// [
OpenSquare,
/// ]
@@ -72,3 +70,13 @@ pub enum Punctuation {
/// "
Hash,
}
+
+pub trait TokenStringExt {
+ fn first_word(&self) -> Option<Token>;
+}
+
+impl TokenStringExt for [Token] {
+ fn first_word(&self) -> Option<Token> {
+ self.iter().find(|v| v.kind.is_word()).copied()
+ }
+}
diff --git a/lt-core/src/spell/dictionary.rs b/lt-core/src/spell/dictionary.rs
index 6941f809d..8db100edd 100644
--- a/lt-core/src/spell/dictionary.rs
+++ b/lt-core/src/spell/dictionary.rs
@@ -1,28 +1,69 @@
-use std::collections::HashSet;
+use std::{collections::HashSet, iter};
+use cached::proc_macro::cached;
+
+#[derive(Debug, Clone)]
pub struct Dictionary {
/// Storing a separate [Vec] for iterations speeds up spellchecking by ~16% at the cost of
/// additional memory.
///
/// This is likely due to increased locality :shrug:.
+ ///
+ /// This list is sorted by word length (i.e. the shortest words are first).
words: Vec<Vec<char>>,
+ /// A lookup list for each word length.
+ /// Each index of this list will return the first index of [`Self::words`] that has a word of
+ /// that length.
+ word_len_starts: Vec<usize>,
word_set: HashSet<Vec<char>>,
}
+#[cached]
+fn cached_inner_new() -> Dictionary {
+ let english_words_raw = include_str!("../../../english_words.txt").replace('\r', "");
+
+ let mut words: Vec<Vec<char>> = english_words_raw
+ .split('\n')
+ .filter(|word| !word.is_empty())
+ .map(|word| word.chars().collect())
+ .collect();
+
+ words.sort_by_key(|a| a.len());
+
+ let mut word_len_starts = vec![0, 0];
+
+ for (index, len) in words.iter().map(Vec::len).enumerate() {
+ if word_len_starts.len() == len {
+ word_len_starts.push(index);
+ }
+ }
+
+ Dictionary {
+ word_set: HashSet::from_iter(words.iter().cloned()),
+ word_len_starts,
+ words,
+ }
+}
+
impl Dictionary {
- pub fn create_from_static() -> Self {
- let english_words_raw = include_str!("../../../english_words.txt").replace('\r', "");
-
- let words: Vec<_> = english_words_raw
- .split('\n')
- .filter(|word| !word.is_empty())
- .map(|word| word.chars().collect())
- .collect();
-
- Self {
- word_set: HashSet::from_iter(words.iter().cloned()),
- words,
+ pub fn new() -> Self {
+ cached_inner_new()
+ }
+
+ /// Iterate over all the words in the dicitonary of a given length
+ pub fn words_with_len_iter(&self, len: usize) -> Box<dyn Iterator<Item = &'_ [char]> + '_> {
+ if len == 0 || len >= self.word_len_starts.len() {
+ return Box::new(iter::empty());
}
+
+ let start = self.word_len_starts[len];
+ let end = if len + 1 == self.word_len_starts.len() {
+ self.words.len()
+ } else {
+ self.word_len_starts[len + 1]
+ };
+
+ Box::new(self.words[start..end].iter().map(|v| v.as_slice()))
}
pub fn words_iter(&self) -> impl Iterator<Item = &'_ [char]> {
@@ -35,3 +76,9 @@ impl Dictionary {
self.word_set.contains(word) || self.word_set.contains(&lowercase)
}
}
+
+impl Default for Dictionary {
+ fn default() -> Self {
+ Self::new()
+ }
+}
diff --git a/lt-core/src/spell/mod.rs b/lt-core/src/spell/mod.rs
index ecdb7fe99..c74766baf 100644
--- a/lt-core/src/spell/mod.rs
+++ b/lt-core/src/spell/mod.rs
@@ -15,31 +15,38 @@ pub fn suggest_correct_spelling<'a>(
let mut buf_a = Vec::with_capacity(53);
let mut buf_b = Vec::with_capacity(53);
- let pruned_words = dictionary
- .words_iter()
- .filter(|word| word.len().abs_diff(misspelled_word.len()) <= max_edit_dist as usize)
- .filter_map(|word| {
- let dist = edit_distance_min_alloc(misspelled_word, word, &mut buf_a, &mut buf_b);
-
- if dist <= max_edit_dist {
- Some((word, dist))
- } else {
- None
- }
- });
+ // The length of the shortest word to look at.
+ let shortest_word_len = if misspelled_word.len() < max_edit_dist as usize {
+ 1
+ } else {
+ misspelled_word.len() - max_edit_dist as usize
+ };
+
+ let words_to_search = (shortest_word_len..misspelled_word.len() + max_edit_dist as usize)
+ .flat_map(|len| dictionary.words_with_len_iter(len));
+
+ let pruned_words = words_to_search.filter_map(|word| {
+ let dist = edit_distance_min_alloc(misspelled_word, word, &mut buf_a, &mut buf_b);
+
+ if dist <= max_edit_dist {
+ Some((word, dist))
+ } else {
+ None
+ }
+ });
let mut found: Vec<(&[char], u8)> = Vec::with_capacity(result_limit);
for (word, dist) in pruned_words {
if found.len() < result_limit {
found.push((word, dist));
- found.sort_by(|a, b| a.1.cmp(&b.1));
+ found.sort_by_key(|a| a.1);
continue;
}
if dist < found[result_limit - 1].1 {
found[result_limit - 1] = (word, dist);
- found.sort_by(|a, b| a.1.cmp(&b.1));
+ found.sort_by_key(|a| a.1);
}
}
@@ -71,13 +78,16 @@ fn edit_distance_min_alloc(
previous_row: &mut Vec<u8>,
current_row: &mut Vec<u8>,
) -> u8 {
- assert!(source.len() <= 255 && target.len() <= 255);
+ if cfg!(debug) {
+ assert!(source.len() <= 255 && target.len() <= 255);
+ }
let row_width = source.len();
let col_height = target.len();
previous_row.clear();
previous_row.extend(0u8..=row_width as u8);
+ // Alright if not zeroed, since we overwrite it anyway
current_row.resize(row_width + 1, 0);
for j in 1..=col_height {
diff --git a/lt-serve/src/main.rs b/lt-serve/src/main.rs
index 078a51657..019386149 100644
--- a/lt-serve/src/main.rs
+++ b/lt-serve/src/main.rs
@@ -2,36 +2,71 @@
use lt_core::{all_linters, Document, FatToken, Lint, Span, Suggestion};
use std::net::SocketAddr;
-use tracing::info;
-
-use axum::{extract::Query, http::StatusCode, routing::get, Json, Router};
+use tokio::time::Instant;
+use tracing::{debug, info, Level};
+use tracing_subscriber::FmtSubscriber;
+
+use axum::{
+ body::Body,
+ http::Request,
+ http::StatusCode,
+ middleware::{self, Next},
+ response::Response,
+ routing::{get, post},
+ Json, Router,
+};
use serde::{Deserialize, Serialize};
#[tokio::main]
async fn main() {
- tracing_subscriber::fmt::init();
+ let subscriber = FmtSubscriber::builder()
+ .with_max_level(Level::DEBUG)
+ .finish();
+
+ tracing::subscriber::set_global_default(subscriber).expect("setting default subscriber failed");
let app = Router::new()
- .route("/", get(root))
- .route("/parse", get(parse_text))
- .route("/lint", get(lint))
- .route("/apply", get(apply_suggestion));
+ .route("/", post(root))
+ .route("/parse", post(parse_text))
+ .route("/lint", post(lint))
+ .route("/apply", post(apply_suggestion))
+ .layer(middleware::from_fn(timing_middleware));
let addr = SocketAddr::from(([127, 0, 0, 1], 3001));
- tracing::debug!("listening on {}", addr);
+ info!("listening on {}", addr);
axum::Server::bind(&addr)
.serve(app.into_make_service())
.await
.unwrap();
}
+async fn timing_middleware(request: Request<Body>, next: Next<Body>) -> Response {
+ let uri = request.uri().clone();
+
+ let start = Instant::now();
+
+ let res = next.run(request).await;
+
+ let end = Instant::now();
+
+ let diff = end - start;
+
+ info!(
+ "Took {} ms to process request at endpoint: {}",
+ diff.as_millis(),
+ uri.path(),
+ );
+
+ res
+}
+
async fn root() -> &'static str {
info!("Hello world");
"Hello, World!"
}
-async fn parse_text(Query(payload): Query<ParseRequest>) -> (StatusCode, Json<ParseResponse>) {
+async fn parse_text(Json(payload): Json<ParseRequest>) -> (StatusCode, Json<ParseResponse>) {
let text = payload.text;
let document = Document::new(&text);
@@ -50,7 +85,7 @@ struct ParseResponse {
pub tokens: Vec<FatToken>,
}
-async fn lint(Query(payload): Query<LintRequest>) -> (StatusCode, Json<LintResponse>) {
+async fn lint(Json(payload): Json<LintRequest>) -> (StatusCode, Json<LintResponse>) {
let text = payload.text;
let document = Document::new(&text);
@@ -71,22 +106,11 @@ struct LintResponse {
}
async fn apply_suggestion(
- Query(payload): Query<ApplySuggestionRequest>,
+ Json(payload): Json<ApplySuggestionRequest>,
) -> (StatusCode, Json<ApplySuggestionResponse>) {
let text = payload.text;
-
- let Ok(SuggestionData { suggestion, span }) = serde_json::from_str(&payload.data) else {
- return (
- StatusCode::BAD_REQUEST,
- Json(ApplySuggestionResponse {
- text: "".to_string(),
- }),
- );
- };
-
let mut document = Document::new(&text);
-
- document.apply_suggestion(&suggestion, span);
+ document.apply_suggestion(&payload.suggestion, payload.span);
(
StatusCode::ACCEPTED,
@@ -99,13 +123,8 @@ async fn apply_suggestion(
#[derive(Deserialize)]
struct ApplySuggestionRequest {
pub text: String,
- pub data: String,
-}
-
-#[derive(Deserialize)]
-struct SuggestionData {
- suggestion: Suggestion,
- span: Span,
+ pub suggestion: Suggestion,
+ pub span: Span,
}
#[derive(Serialize)]
diff --git a/web/src/lib/Highlights.svelte b/web/src/lib/Highlights.svelte
index a0121f6e5..4aebe12df 100644
--- a/web/src/lib/Highlights.svelte
+++ b/web/src/lib/Highlights.svelte
@@ -1,50 +1,8 @@
<script lang="ts">
+ import type { Lint, Token, TokenKind } from '$lib/analysis';
+ import { parseText, contentToString } from '$lib/analysis';
export let content: string;
- interface ParseResponse {
- tokens: Token[];
- }
-
- interface Token {
- content: string[];
- kind: TokenKind;
- }
-
- type TokenKind =
- | { kind: 'Word' }
- | { kind: 'Punctuation'; value: Punctuation }
- | { kind: 'Number'; value: number }
- | { kind: 'Space'; value: number }
- | { kind: 'Newline'; value: number };
-
- type Punctuation =
- | 'Period'
- | 'Bang'
- | 'Question'
- | 'Colon'
- | 'Semicolon'
- | 'Quote'
- | 'Comma'
- | 'Hyphen'
- | 'Apostrophe'
- | 'OpenSquare'
- | 'CloseSquare'
- | 'OpenRound'
- | 'CloseRound'
- | 'Hash';
-
- async function parseText(text: string): Promise<Token[]> {
- const req = await fetch(`/parse?text=${encodeURIComponent(text)}`);
-
- const res: ParseResponse = await req.json();
-
- return res.tokens;
- }
-
- function contentToString(content: string[]): string {
- return content.reduce((p, c) => p + c, '');
- }
-
let tokens: Token[] = [];
$: parseText(content).then((parsedTokens) => (tokens = parsedTokens));
diff --git a/web/src/lib/analysis.ts b/web/src/lib/analysis.ts
new file mode 100644
index 000000000..b3ce8e56a
--- /dev/null
+++ b/web/src/lib/analysis.ts
@@ -0,0 +1,99 @@
+export interface ParseResponse {
+ tokens: Token[];
+}
+
+export interface Token {
+ content: string[];
+ kind: TokenKind;
+}
+
+export type TokenKind =
+ | { kind: 'Word' }
+ | { kind: 'Punctuation'; value: Punctuation }
+ | { kind: 'Number'; value: number }
+ | { kind: 'Space'; value: number }
+ | { kind: 'Newline'; value: number };
+
+export type Punctuation =
+ | 'Period'
+ | 'Bang'
+ | 'Question'
+ | 'Colon'
+ | 'Semicolon'
+ | 'Quote'
+ | 'Comma'
+ | 'Hyphen'
+ | 'Apostrophe'
+ | 'OpenSquare'
+ | 'CloseSquare'
+ | 'OpenRound'
+ | 'CloseRound'
+ | 'Hash';
+
+export async function parseText(text: string): Promise<Token[]> {
+ const req = await fetch('/parse', {
+ method: 'POST',
+ body: JSON.stringify({ text }),
+ headers: {
+ 'CONTENT-TYPE': 'application/json'
+ }
+ });
+
+ const res: ParseResponse = await req.json();
+
+ return res.tokens;
+}
+
+export function contentToString(content: string[]): string {
+ return content.reduce((p, c) => p + c, '');
+}
+
+export interface LintResponse {
+ lints: Lint[];
+}
+
+export interface Lint {
+ span: Span;
+ lint_kind: 'Capitalization' | 'Spelling';
+ suggestions: Suggestion[];
+}
+
+export interface Suggestion {
+ ReplaceWith: string[];
+}
+
+export interface Span {
+ start: number;
+ end: number;
+}
+
+export async function lintText(text: string): Promise<Lint[]> {
+ const req = await fetch(`/lint`, {
+ method: 'POST',
+ body: JSON.stringify({ text }),
+ headers: {
+ 'CONTENT-TYPE': 'application/json'
+ }
+ });
+
+ const res: LintResponse = await req.json();
+
+ return res.lints;
+}
+
+export async function applySuggestion(
+ text: string,
+ suggestion: Suggestion,
+ span: Span
+): Promise<string> {
+ const req = await fetch(`/apply`, {
+ method: 'POST',
+ body: JSON.stringify({ text, suggestion, span }),
+ headers: {
+ 'CONTENT-TYPE': 'application/json'
+ }
+ });
+
+ const res = await req.json();
+ return res.text;
+}
diff --git a/web/src/routes/+page.svelte b/web/src/routes/+page.svelte
index afd26f986..38acd6592 100644
--- a/web/src/routes/+page.svelte
+++ b/web/src/routes/+page.svelte
@@ -3,53 +3,22 @@
import alice from '../../../alice.txt?raw';
import Highlights from '$lib/Highlights.svelte';
import { Button } from 'flowbite-svelte';
+ import { lintText, applySuggestion } from '$lib/analysis';
+ import type { Lint } from '$lib/analysis';
let content = alice;
let editor: HTMLTextAreaElement;
- interface LintResponse {
- lints: Lint[];
- }
-
- interface Lint {
- span: Span;
- lint_kind: 'Capitalization' | 'Spelling';
- suggestions: Suggestion[];
- }
-
- interface Suggestion {
- ReplaceWith: string[];
- }
-
- interface Span {
- start: number;
- end: number;
- }
-
- async function lintText(text: string): Promise<Lint[]> {
- const req = await fetch(`/lint?text=${encodeURIComponent(text)}`);
-
- const res: LintResponse = await req.json();
-
- return res.lints;
- }
+ let lints: Lint[] = [];
- async function applySuggestion(
- text: string,
- suggestion: Suggestion,
- span: Span
- ): Promise<string> {
- const req = await fetch(
- `/apply?text=${encodeURIComponent(text)}&data=${JSON.stringify({ span, suggestion })}`
- );
+ $: lintText(content).then((newLints) => (lints = newLints));
- const res = await req.json();
- return res.text;
+ function ping() {
+ lintText(content).then((newLints) => (lints = newLints));
+ setTimeout(ping, 1000);
}
- let lints: Lint[] = [];
-
- $: lintText(content).then((newLints) => (lints = newLints));
+ ping();
</script>
<div class="flex flex-row w-full h-screen">