Skip to content

Offline, privacy-first grammar checker. Fast, open-source, Rust-powered

feat(fst-dict): create build script to generate map file

Grant Lemonsd9b361dcaa61
diff --git a/.gitignore b/.gitignore
index bd44edec9..6f572f0bc 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,2 +1,3 @@
target
build
+*.fst
diff --git a/Cargo.lock b/Cargo.lock
index a81815b9f..79ff55e99 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -420,6 +420,15 @@ dependencies = [
"percent-encoding",
]
+[[package]]
+name = "fst"
+version = "0.4.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7ab85b9b05e3978cc9a9cf8fea7f01b494e1a09ed3037e16ba39edc7a29eb61a"
+dependencies = [
+ "utf8-ranges",
+]
+
[[package]]
name = "futures"
version = "0.3.31"
@@ -590,9 +599,11 @@ version = "0.12.0"
dependencies = [
"blanket",
"criterion",
+ "fst",
"hashbrown 0.15.0",
"is-macro",
"itertools 0.13.0",
+ "memmap",
"ordered-float",
"paste",
"pulldown-cmark",
@@ -985,6 +996,16 @@ version = "2.7.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3"
+[[package]]
+name = "memmap"
+version = "0.7.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6585fd95e7bb50d6cc31e20d4cf9afb4e2ba16c5846fc76793f11218da9c475b"
+dependencies = [
+ "libc",
+ "winapi",
+]
+
[[package]]
name = "miniz_oxide"
version = "0.8.0"
@@ -1867,6 +1888,12 @@ version = "1.0.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c8232dd3cdaed5356e0f716d285e4b40b932ac434100fe9b7e0e8e935b9e6246"
+[[package]]
+name = "utf8-ranges"
+version = "1.0.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7fcfc827f90e53a02eaef5e535ee14266c1d569214c6aa70133a624d8a3164ba"
+
[[package]]
name = "utf8_iter"
version = "1.0.4"
diff --git a/harper-core/Cargo.toml b/harper-core/Cargo.toml
index d4ba8cc06..81ad19589 100644
--- a/harper-core/Cargo.toml
+++ b/harper-core/Cargo.toml
@@ -9,9 +9,11 @@ repository = "https://github.com/elijah-potter/harper"
[dependencies]
blanket = "0.4.0"
+fst = { version = "0.4.7", features = ["levenshtein"] }
hashbrown = { version = "0.15.0", features = ["serde"] }
is-macro = "0.3.6"
itertools = "0.13.0"
+memmap = "0.7.0"
ordered-float = { version = "4.5.0", features = ["serde"] }
paste = "1.0.14"
pulldown-cmark = "0.12.2"
@@ -26,6 +28,9 @@ unicode-width = "0.2.0"
criterion = { version = "0.5.1", default-features = false }
rand = "0.8.5"
+[build-dependencies]
+fst = { version = "0.4.7", features = ["levenshtein"] }
+
[[bench]]
name = "parse_demo"
harness = false
diff --git a/harper-core/build.rs b/harper-core/build.rs
new file mode 100644
index 000000000..87020e2af
--- /dev/null
+++ b/harper-core/build.rs
@@ -0,0 +1,42 @@
+use std::fs::File;
+use std::io;
+
+use fst::MapBuilder;
+
+pub fn parse_word_list(source: &str) -> Vec<String> {
+ let mut lines = source.lines();
+
+ let approx_item_count = lines.next().unwrap().parse().unwrap();
+ let mut words = Vec::with_capacity(approx_item_count);
+
+ for line in lines {
+ if let Some((word, _attributes)) = line.split_once('/') {
+ words.push(word.chars().collect())
+ } else {
+ words.push(line.chars().collect())
+ }
+ }
+
+ words
+}
+
+fn main() {
+ let mut wtr = io::BufWriter::new(File::create("dictionary.fst").unwrap());
+ let mut build = MapBuilder::new(wtr).unwrap();
+
+ let mut word_list: Vec<String> = parse_word_list(include_str!("dictionary.dict"));
+ word_list.sort();
+ word_list.dedup();
+
+ // Using u64 shouldn't pose any issues since I don't think the English
+ // language has that many words
+ word_list
+ .iter()
+ .enumerate()
+ .for_each(|(i, s)| build.insert(s, i as u64).unwrap());
+
+ build.finish().expect("Unable to build map of dictionary!");
+
+ println!("cargo::rerun-if-changed=build.rs");
+ println!("cargo::rerun-if-changed=dictionary.dict");
+}