Offline, privacy-first grammar checker. Fast, open-source, Rust-powered
feat(fst-dict): create build script to generate map file
Grant Lemonsd9b361dcaa61
diff --git a/.gitignore b/.gitignoreindex bd44edec9..6f572f0bc 100644--- a/.gitignore+++ b/.gitignore@@ -1,2 +1,3 @@targetbuild+*.fstdiff --git a/Cargo.lock b/Cargo.lockindex a81815b9f..79ff55e99 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -420,6 +420,15 @@ dependencies = ["percent-encoding",]+[[package]]+name = "fst"+version = "0.4.7"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "7ab85b9b05e3978cc9a9cf8fea7f01b494e1a09ed3037e16ba39edc7a29eb61a"+dependencies = [+ "utf8-ranges",+]+[[package]]name = "futures"version = "0.3.31"@@ -590,9 +599,11 @@ version = "0.12.0"dependencies = ["blanket","criterion",+ "fst","hashbrown 0.15.0","is-macro","itertools 0.13.0",+ "memmap","ordered-float","paste","pulldown-cmark",@@ -985,6 +996,16 @@ version = "2.7.4"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3"+[[package]]+name = "memmap"+version = "0.7.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "6585fd95e7bb50d6cc31e20d4cf9afb4e2ba16c5846fc76793f11218da9c475b"+dependencies = [+ "libc",+ "winapi",+]+[[package]]name = "miniz_oxide"version = "0.8.0"@@ -1867,6 +1888,12 @@ version = "1.0.5"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "c8232dd3cdaed5356e0f716d285e4b40b932ac434100fe9b7e0e8e935b9e6246"+[[package]]+name = "utf8-ranges"+version = "1.0.5"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "7fcfc827f90e53a02eaef5e535ee14266c1d569214c6aa70133a624d8a3164ba"+[[package]]name = "utf8_iter"version = "1.0.4"diff --git a/harper-core/Cargo.toml b/harper-core/Cargo.tomlindex d4ba8cc06..81ad19589 100644--- a/harper-core/Cargo.toml+++ b/harper-core/Cargo.toml@@ -9,9 +9,11 @@ repository = "https://github.com/elijah-potter/harper"[dependencies]blanket = "0.4.0"+fst = { version = "0.4.7", features = ["levenshtein"] }hashbrown = { version = "0.15.0", features = ["serde"] }is-macro = "0.3.6"itertools = "0.13.0"+memmap = "0.7.0"ordered-float = { version = "4.5.0", features = ["serde"] }paste = "1.0.14"pulldown-cmark = "0.12.2"@@ -26,6 +28,9 @@ unicode-width = "0.2.0"criterion = { version = "0.5.1", default-features = false }rand = "0.8.5"+[build-dependencies]+fst = { version = "0.4.7", features = ["levenshtein"] }+[[bench]]name = "parse_demo"harness = falsediff --git a/harper-core/build.rs b/harper-core/build.rsnew file mode 100644index 000000000..87020e2af--- /dev/null+++ b/harper-core/build.rs@@ -0,0 +1,42 @@+use std::fs::File;+use std::io;++use fst::MapBuilder;++pub fn parse_word_list(source: &str) -> Vec<String> {+ let mut lines = source.lines();++ let approx_item_count = lines.next().unwrap().parse().unwrap();+ let mut words = Vec::with_capacity(approx_item_count);++ for line in lines {+ if let Some((word, _attributes)) = line.split_once('/') {+ words.push(word.chars().collect())+ } else {+ words.push(line.chars().collect())+ }+ }++ words+}++fn main() {+ let mut wtr = io::BufWriter::new(File::create("dictionary.fst").unwrap());+ let mut build = MapBuilder::new(wtr).unwrap();++ let mut word_list: Vec<String> = parse_word_list(include_str!("dictionary.dict"));+ word_list.sort();+ word_list.dedup();++ // Using u64 shouldn't pose any issues since I don't think the English+ // language has that many words+ word_list+ .iter()+ .enumerate()+ .for_each(|(i, s)| build.insert(s, i as u64).unwrap());++ build.finish().expect("Unable to build map of dictionary!");++ println!("cargo::rerun-if-changed=build.rs");+ println!("cargo::rerun-if-changed=dictionary.dict");+}